From 13cff6354ba1bf26e05d6f8177959cd43a31b12c Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Mon, 17 Aug 2020 20:58:54 -0300 Subject: [PATCH 1/4] Add support to define which playbook tests to execute with pytest. pytest provide the means to skip tests based on patterns, but writing these patterns for ansible-freeipa might not be feasible. This PR allows the selection of playbook tests and modules that will be executed with pytest using the environmentt variables IPA_ENABLED_TESTS IPA_ENABLED_MODULES, IPA_DISABLED_TESTS or IPA_DISABLED_MODULES. When using IPA_ENABLED_MODULES, all modules will be disabled, and only the modules in the enabled list will be tested. If using the test filter, IPA_ENABLED_TESTS, all tests are disabled, unless they are in the enabled test lists. If the IPA_DISABLED_* version is used, tests and modules are enabled by default, and the list is used to disable the module or specific test. To disable a test or module in Azure CI, edit the file `tests/azure/variables` and add the desired tests or modules to the parameter variables `enabled_modules`, 'enabled_tests`, `disabled_tests` or `disable_modules`. Note that, if added to the `master` branch, this will affect the tests for every pipeline that it is include (including 'nightly'), so it should be used with care. It can be used with TEMP commits to enable only the desired tests, speeding up upstream tests. --- tests/README.md | 19 ++++++++ tests/azure/templates/playbook_tests.yml | 7 ++- tests/azure/templates/pytest_tests.yml | 4 ++ tests/azure/templates/variables.yaml | 18 +++++++ tests/test_playbook_runs.py | 24 +++++---- tests/utils.py | 62 ++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 tests/azure/templates/variables.yaml diff --git a/tests/README.md b/tests/README.md index 631a37c8..65ba9762 100644 --- a/tests/README.md +++ b/tests/README.md @@ -63,6 +63,24 @@ IPA_SERVER_HOST= pytest -rs For a complete list of options check `pytest --help`. +### Disabling and enabling playbook tests + +Sometimes it is useful to enable or disable specific playbook tests. To only run a subset of modules or tests, use the variables IPA_ENABLED_MODULES and IPA ENABLED_TESTS, to define a comma-separated list of modules or tests to be enabled. Any test or module not in the list will not be executed. For example, to run only `sudorule` and `sudocmd` tests: + +``` +IPA_ENABLE_MODULES="sudorule,sudocmd" IPA_SERVER_HOST= pytest +``` + +If all but a few selected tests are to be executed, use the IPA_DISABLED_MODULES or IPA_DISABLED_TESTS. For example, to run all, but "test_service_certificate" test: + +``` +IPA_DISABLED_TESTS=test_service_certificate IPA_SERVER_HOST= pytest +``` + +If none of this variables are defined, all tests will be executed. + +To configure the tests that will run for your pull request, add a TEMP commit, with the configuration defined in the file `tests/azure/templates/variables.yml`. Set the variables `ipa_enable_modules`, `ipa_enable_tests`, `ipa_disable_modules`, and `ipa_disable_tests`, in the same way as the equivalent environment variables. + ### Types of tests #### Playbook tests @@ -119,6 +137,7 @@ molecule destroy -s c8s See [Running the tests](#running-the-tests) section for more information on available options. + ## Upcoming/desired improvements: * A script to pre-config the complete test environment using virsh. diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml index 09775045..c838cbc3 100644 --- a/tests/azure/templates/playbook_tests.yml +++ b/tests/azure/templates/playbook_tests.yml @@ -18,11 +18,12 @@ parameters: - name: build_number type: string - jobs: - job: Test_Group${{ parameters.group_number }} displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) timeoutInMinutes: 120 + variables: + - template: variables.yaml steps: - task: UsePythonVersion@0 inputs: @@ -63,6 +64,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true + IPA_DISABLED_MODULES: ${{ variables.disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.enabled_tests }} - task: PublishTestResults@2 inputs: diff --git a/tests/azure/templates/pytest_tests.yml b/tests/azure/templates/pytest_tests.yml index 92783c52..48a8cb0f 100644 --- a/tests/azure/templates/pytest_tests.yml +++ b/tests/azure/templates/pytest_tests.yml @@ -53,6 +53,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true + IPA_DISABLED_MODULES: ${{ variables.disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.enabled_tests }} - task: PublishTestResults@2 inputs: diff --git a/tests/azure/templates/variables.yaml b/tests/azure/templates/variables.yaml new file mode 100644 index 00000000..109ec96d --- /dev/null +++ b/tests/azure/templates/variables.yaml @@ -0,0 +1,18 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + # ipa_enabled_modules: >- + # ipa_enabled_tests: >- + # ipa_disabled_modules: >- + # ipa_disabled_tests: >- diff --git a/tests/test_playbook_runs.py b/tests/test_playbook_runs.py index 4a9c24b7..174a10f9 100644 --- a/tests/test_playbook_runs.py +++ b/tests/test_playbook_runs.py @@ -24,11 +24,12 @@ import functools from unittest import TestCase -from utils import get_test_playbooks, get_server_host, run_playbook +from utils import get_test_playbooks, get_skip_conditions, run_playbook -def prepare_test(test_name, test_path): - """Decorator for the tests generated automatically from playbooks. +def prepare_test(testname, testpath): + """ + Decorate tests generated automatically from playbooks. Injects 2 arguments to the test (`test_path` and `test_name`) and name the test method using test name (to ensure test reports are useful). @@ -36,13 +37,13 @@ def prepare_test(test_name, test_path): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): - kwargs["test_path"] = test_path - kwargs["test_name"] = test_name + kwargs["test_path"] = testpath + kwargs["test_name"] = testname return func(*args, **kwargs) return wrapper - decorator.__name__ = test_name + decorator.__name__ = testname return decorator @@ -50,18 +51,21 @@ def prepare_test(test_name, test_path): # test_* methods. for test_dir_name, playbooks_in_dir in get_test_playbooks().items(): _tests = {} + for playbook in playbooks_in_dir: test_name = playbook["name"].replace("-", "_") test_path = playbook["path"] - @pytest.mark.skipif( - not get_server_host(), - reason="Environment variable IPA_SERVER_HOST must be set", - ) + skip = get_skip_conditions(test_dir_name, test_name) or {} + + # pylint: disable=W0621,W0640,W0613 + @pytest.mark.skipif(**skip) @pytest.mark.playbook @prepare_test(test_name, test_path) def method(self, test_path, test_name): run_playbook(test_path) + # pylint: enable=W0621,W0640,W0613 _tests[test_name] = method + globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,) diff --git a/tests/utils.py b/tests/utils.py index d681b0d6..ae7cd48d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -45,6 +45,68 @@ def get_server_host(): return os.getenv("IPA_SERVER_HOST") +def get_disabled_test(group_name, test_name): + disabled_modules = [ + disabled.strip() + for disabled in os.environ.get("IPA_DISABLED_MODULES", "").split(",") + ] + disabled_tests = [ + disabled.strip() + for disabled in os.environ.get("IPA_DISABLED_TESTS", "").split(",") + if disabled.strip() + ] + + if not any([disabled_modules, disabled_tests]): + return False + + return group_name in disabled_modules or test_name in disabled_tests + + +def get_enabled_test(group_name, test_name): + enabled_modules = [ + enabled.strip() + for enabled in os.environ.get("IPA_ENABLED_MODULES", "").split(":") + if enabled.strip() + ] + enabled_tests = [ + enabled.strip() + for enabled in os.environ.get("IPA_ENABLED_TESTS", "").split(":") + if enabled.strip() + ] + + if not any([enabled_modules, enabled_tests]): + return True + + group_enabled = group_name in enabled_modules + test_enabled = test_name in enabled_tests + + return group_enabled or test_enabled + + +def get_skip_conditions(group_name, test_name): + """ + Check tests that need to be skipped. + + The return is a dict containing `condition` and `reason`. For the test + to be skipped, `condition` must be True, if it is `False`, the test is + to be skipped. Although "reason" must be always provided, it can be + `None` if `condition` is True. + """ + if not get_server_host(): + return { + "condition": True, + "reason": "Environment variable IPA_SERVER_HOST must be set", + } + + if not get_enabled_test(group_name, test_name): + return {"condition": True, "reason": "Test not configured to run"} + + if get_disabled_test(group_name, test_name): + return {"condition": True, "reason": "Test configured to not run"} + + return {"condition": False, "reason": "Test will run."} + + def get_inventory_content(): """Create the content of an inventory file for a test run.""" ipa_server_host = get_server_host() From 914e4879f8f39f6ec666f5eb86c5508ac27f53f6 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Tue, 30 Nov 2021 19:18:17 -0300 Subject: [PATCH 2/4] tests/utils.py: Fix pylint issues. --- tests/sanity/ignore-2.12.txt | 1 - tests/utils.py | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/sanity/ignore-2.12.txt b/tests/sanity/ignore-2.12.txt index b08a6ffa..ee85fb0a 100644 --- a/tests/sanity/ignore-2.12.txt +++ b/tests/sanity/ignore-2.12.txt @@ -43,7 +43,6 @@ tests/sanity/sanity.sh shebang!skip tests/user/users.sh shebang!skip tests/user/users_absent.sh shebang!skip tests/utils.py pylint:ansible-format-automatic-specification -tests/utils.py pylint:subprocess-run-check utils/ansible-doc-test shebang!skip utils/ansible-ipa-client-install shebang!skip utils/ansible-ipa-replica-install shebang!skip diff --git a/tests/utils.py b/tests/utils.py index ae7cd48d..db22f973 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -174,6 +174,7 @@ def _run_playbook(playbook): inventory_file.name, playbook, ] + # pylint: disable=subprocess-run-check process = subprocess.run( cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -300,11 +301,13 @@ class AnsibleFreeIPATestCase(TestCase): host_connection_info, ssh_identity_file=ssh_identity_file, ) - def run_playbook(self, playbook, allow_failures=False): + @staticmethod + def run_playbook(playbook, allow_failures=False): return run_playbook(playbook, allow_failures) - def run_playbook_with_exp_msg(self, playbook, expected_msg): - result = self.run_playbook(playbook, allow_failures=True) + @staticmethod + def run_playbook_with_exp_msg(playbook, expected_msg): + result = run_playbook(playbook, allow_failures=True) assert ( expected_msg in result.stdout.decode("utf8") or From 757b89dfaea909eb26c00aee20a1c0512f65c55a Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Tue, 14 Jun 2022 21:25:07 -0300 Subject: [PATCH 3/4] upstream tests: Disable dnsconfig and dnsforwardzone Due to an issue with IPA in Fedora 36, dnsconfig and dnsforwardzone plugin tests must be disabled. See FreeIPA issue: https://pagure.io/freeipa/issue/9158 --- tests/azure/templates/variables.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/azure/templates/variables.yaml b/tests/azure/templates/variables.yaml index 109ec96d..f200416b 100644 --- a/tests/azure/templates/variables.yaml +++ b/tests/azure/templates/variables.yaml @@ -14,5 +14,7 @@ variables: # ipa_enabled_modules: >- # ipa_enabled_tests: >- - # ipa_disabled_modules: >- + ipa_disabled_modules: >- + dnsconfig, + dnsforwardzone, # ipa_disabled_tests: >- From ac92ed1408e6150c60fd2ea34430194dd3bcf39e Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Wed, 15 Jun 2022 09:53:32 -0300 Subject: [PATCH 4/4] fixup! Add support to define which playbook tests to execute with pytest. --- tests/azure/templates/playbook_tests.yml | 8 ++++---- tests/azure/templates/pytest_tests.yml | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml index c838cbc3..3fb15ad3 100644 --- a/tests/azure/templates/playbook_tests.yml +++ b/tests/azure/templates/playbook_tests.yml @@ -64,10 +64,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true - IPA_DISABLED_MODULES: ${{ variables.disabled_modules }} - IPA_DISABLED_TESTS: ${{ variables.disabled_tests }} - IPA_ENABLED_MODULES: ${{ variables.enabled_modules }} - IPA_ENABLED_TESTS: ${{ variables.enabled_tests }} + IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} - task: PublishTestResults@2 inputs: diff --git a/tests/azure/templates/pytest_tests.yml b/tests/azure/templates/pytest_tests.yml index 48a8cb0f..f25ce2a7 100644 --- a/tests/azure/templates/pytest_tests.yml +++ b/tests/azure/templates/pytest_tests.yml @@ -16,6 +16,8 @@ jobs: - job: Test_PyTests displayName: Run pytests on ${{ parameters.scenario }} timeoutInMinutes: 120 + variables: + - template: variables.yaml steps: - task: UsePythonVersion@0 inputs: @@ -53,10 +55,10 @@ jobs: env: IPA_SERVER_HOST: ${{ parameters.scenario }} RUN_TESTS_IN_DOCKER: true - IPA_DISABLED_MODULES: ${{ variables.disabled_modules }} - IPA_DISABLED_TESTS: ${{ variables.disabled_tests }} - IPA_ENABLED_MODULES: ${{ variables.enabled_modules }} - IPA_ENABLED_TESTS: ${{ variables.enabled_tests }} + IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} - task: PublishTestResults@2 inputs: