From edccf70bf63932dfdf443f5f8741e95f8897a63c Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Mon, 22 Aug 2022 16:03:07 -0300 Subject: [PATCH 1/3] upstream ci: Add support for distro specific test configuration. Sometimes, mostly due do differences in package versions, there are some tests that fail on a single distribution which cannot be fixed by ansible-freeipa, requiring that the offending package is fixed. To keep tests running succesfully we have options to disable the failing tests, but this changes are globally applied, meaning that, by disabling a test, it is disable in all tested distributions. This patch allows tests to be enabled or disabled for a specific distribution, by setting the configuration on the 'variable' template for the specific testing scenario. --- tests/azure/templates/galaxy_script.yml | 1 + tests/azure/templates/playbook_tests.yml | 1 + tests/azure/templates/pytest_tests.yml | 1 + tests/azure/templates/variables.yaml | 10 ++++++---- tests/azure/templates/variables_c8s.yaml | 19 ++++++++++++++++++ tests/azure/templates/variables_c9s.yaml | 19 ++++++++++++++++++ tests/azure/templates/variables_centos-7.yaml | 19 ++++++++++++++++++ .../templates/variables_fedora-latest.yaml | 20 +++++++++++++++++++ .../templates/variables_fedora-rawhide.yaml | 19 ++++++++++++++++++ 9 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 tests/azure/templates/variables_c8s.yaml create mode 100644 tests/azure/templates/variables_c9s.yaml create mode 100644 tests/azure/templates/variables_centos-7.yaml create mode 100644 tests/azure/templates/variables_fedora-latest.yaml create mode 100644 tests/azure/templates/variables_fedora-rawhide.yaml diff --git a/tests/azure/templates/galaxy_script.yml b/tests/azure/templates/galaxy_script.yml index b6694818..bf14238c 100644 --- a/tests/azure/templates/galaxy_script.yml +++ b/tests/azure/templates/galaxy_script.yml @@ -25,6 +25,7 @@ jobs: timeoutInMinutes: 120 variables: - template: variables.yaml + - template: variables_${{ parameters.scenario }}.yaml steps: - task: UsePythonVersion@0 inputs: diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml index 3fb15ad3..abbdf8c0 100644 --- a/tests/azure/templates/playbook_tests.yml +++ b/tests/azure/templates/playbook_tests.yml @@ -24,6 +24,7 @@ jobs: timeoutInMinutes: 120 variables: - template: variables.yaml + - template: variables_${{ parameters.scenario }}.yaml steps: - task: UsePythonVersion@0 inputs: diff --git a/tests/azure/templates/pytest_tests.yml b/tests/azure/templates/pytest_tests.yml index f25ce2a7..6afd5890 100644 --- a/tests/azure/templates/pytest_tests.yml +++ b/tests/azure/templates/pytest_tests.yml @@ -18,6 +18,7 @@ jobs: timeoutInMinutes: 120 variables: - template: variables.yaml + - template: variables_${{ parameters.scenario }}.yaml steps: - task: UsePythonVersion@0 inputs: diff --git a/tests/azure/templates/variables.yaml b/tests/azure/templates/variables.yaml index b6e16868..3cdadc03 100644 --- a/tests/azure/templates/variables.yaml +++ b/tests/azure/templates/variables.yaml @@ -3,6 +3,9 @@ # For easier management of items to enable/disable, # use one test/module on each line, followed by a comma. # +# If no variable is to be set, add 'empty: true', as the +# 'variables' dict cannot be empty. +# # Example: # # disabled_modules: >- @@ -12,9 +15,8 @@ # --- variables: + empty: true # ipa_enabled_modules: >- # ipa_enabled_tests: >- - ipa_disabled_modules: >- - dnsconfig, - ipa_disabled_tests: >- - test_dnsconfig_forwarders_ports + # ipa_disabled_modules: >- + # ipa_disabled_tests: >- diff --git a/tests/azure/templates/variables_c8s.yaml b/tests/azure/templates/variables_c8s.yaml new file mode 100644 index 00000000..586d5ecb --- /dev/null +++ b/tests/azure/templates/variables_c8s.yaml @@ -0,0 +1,19 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + empty: true +# ipa_enabled_modules: >- +# ipa_enabled_tests: >- +# ipa_disabled_modules: >- +# ipa_disabled_tests: >- diff --git a/tests/azure/templates/variables_c9s.yaml b/tests/azure/templates/variables_c9s.yaml new file mode 100644 index 00000000..586d5ecb --- /dev/null +++ b/tests/azure/templates/variables_c9s.yaml @@ -0,0 +1,19 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + empty: true +# ipa_enabled_modules: >- +# ipa_enabled_tests: >- +# ipa_disabled_modules: >- +# ipa_disabled_tests: >- diff --git a/tests/azure/templates/variables_centos-7.yaml b/tests/azure/templates/variables_centos-7.yaml new file mode 100644 index 00000000..586d5ecb --- /dev/null +++ b/tests/azure/templates/variables_centos-7.yaml @@ -0,0 +1,19 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + empty: true +# ipa_enabled_modules: >- +# ipa_enabled_tests: >- +# ipa_disabled_modules: >- +# ipa_disabled_tests: >- diff --git a/tests/azure/templates/variables_fedora-latest.yaml b/tests/azure/templates/variables_fedora-latest.yaml new file mode 100644 index 00000000..04fb6d29 --- /dev/null +++ b/tests/azure/templates/variables_fedora-latest.yaml @@ -0,0 +1,20 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + # ipa_enabled_modules: >- + # ipa_enabled_tests: >- + ipa_disabled_modules: >- + dnsforwardzone, + ipa_disabled_tests: >- + test_dnsconfig_forwarders_ports diff --git a/tests/azure/templates/variables_fedora-rawhide.yaml b/tests/azure/templates/variables_fedora-rawhide.yaml new file mode 100644 index 00000000..d828bccb --- /dev/null +++ b/tests/azure/templates/variables_fedora-rawhide.yaml @@ -0,0 +1,19 @@ +# +# Variables must be defined as comma separated lists. +# For easier management of items to enable/disable, +# use one test/module on each line, followed by a comma. +# +# Example: +# +# disabled_modules: >- +# dnsconfig, +# group, +# hostgroup +# +--- +variables: + empty: true + # ipa_enabled_modules: >- + # ipa_enabled_tests: >- + # ipa_disabled_modules: >- + # ipa_disabled_tests: >- From 3216f8df370395c3260ac656d738ba09f4ff0bce Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Thu, 23 Jun 2022 16:59:26 -0300 Subject: [PATCH 2/3] upstream ci: Avoid scheduling tests that will not be executed. Currently, all tests are scheduled to execution, even those that are not executed due to being absent from the list of enabled tests configured in the IPA_ENABLED_* variables. The tests that are not executed are marked 'skipped'. This patch change this behavior by not scheduling tests that are not configured to be executed. It means that tests not the IPA_DISABLED_* lists are not skipped anymore, but not scheduled to be executed. If any test is in IPA_ENABLED_* lists, only those tests are marked for execution. A side effect is that there is no visual feedback on which tests were not executed, as disabled tests are not evaluated anymore. Also, when IPA_SERVER_HOST was not set, all tests were skipped, but an error should raised in this case, as there are no hosts to run the tests against. This patch modifies this behavior to fail the test with an exception if IPA_SERVER_HOST is not set. --- tests/test_playbook_runs.py | 29 ++++++++++++++++++----------- tests/utils.py | 24 ------------------------ 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/tests/test_playbook_runs.py b/tests/test_playbook_runs.py index 174a10f9..a67444d6 100644 --- a/tests/test_playbook_runs.py +++ b/tests/test_playbook_runs.py @@ -24,7 +24,10 @@ import functools from unittest import TestCase -from utils import get_test_playbooks, get_skip_conditions, run_playbook +from utils import ( + get_test_playbooks, get_server_host, run_playbook, get_enabled_test, + get_disabled_test +) def prepare_test(testname, testpath): @@ -47,6 +50,9 @@ def prepare_test(testname, testpath): return decorator +if not get_server_host(): + raise RuntimeError("IPA_SERVER_HOST is not set.") + # Dynamically create the TestCase classes with respective # test_* methods. for test_dir_name, playbooks_in_dir in get_test_playbooks().items(): @@ -56,16 +62,17 @@ for test_dir_name, playbooks_in_dir in get_test_playbooks().items(): test_name = playbook["name"].replace("-", "_") test_path = playbook["path"] - skip = get_skip_conditions(test_dir_name, test_name) or {} + if ( + get_enabled_test(test_dir_name, test_name) + and not get_disabled_test(test_dir_name, test_name) + ): + # pylint: disable=W0621,W0640,W0613 + @pytest.mark.playbook + @prepare_test(test_name, test_path) + def method(self, test_path, test_name): + run_playbook(test_path) + # pylint: enable=W0621,W0640,W0613 - # pylint: disable=W0621,W0640,W0613 - @pytest.mark.skipif(**skip) - @pytest.mark.playbook - @prepare_test(test_name, test_path) - def method(self, test_path, test_name): - run_playbook(test_path) - # pylint: enable=W0621,W0640,W0613 - - _tests[test_name] = method + _tests[test_name] = method globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,) diff --git a/tests/utils.py b/tests/utils.py index c7e630a5..36dac9cf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -84,30 +84,6 @@ def get_enabled_test(group_name, test_name): return group_enabled or test_enabled -def get_skip_conditions(group_name, test_name): - """ - Check tests that need to be skipped. - - The return is a dict containing `condition` and `reason`. For the test - to be skipped, `condition` must be True, if it is `False`, the test is - to be skipped. Although "reason" must be always provided, it can be - `None` if `condition` is True. - """ - if not get_server_host(): - return { - "condition": True, - "reason": "Environment variable IPA_SERVER_HOST must be set", - } - - if not get_enabled_test(group_name, test_name): - return {"condition": True, "reason": "Test not configured to run"} - - if get_disabled_test(group_name, test_name): - return {"condition": True, "reason": "Test configured to not run"} - - return {"condition": False, "reason": "Test will run."} - - def get_inventory_content(): """Create the content of an inventory file for a test run.""" ipa_server_host = get_server_host() From abef329b8aa71aea5a2fe361c858bee002467382 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Wed, 24 Aug 2022 16:08:14 -0300 Subject: [PATCH 3/3] upstream ci: Add step to display scenario configuration Since test configuration can vary in different scenarios (test images) this patch adds a script to list the scenarios configuration, and a step to the playbook test jobs to display the scenario configuration. --- tests/azure/templates/galaxy_script.yml | 4 + tests/azure/templates/playbook_tests.yml | 4 + tests/sanity/ignore-2.12.txt | 1 + utils/check_test_configuration.py | 110 +++++++++++++++++++++++ 4 files changed, 119 insertions(+) create mode 100755 utils/check_test_configuration.py diff --git a/tests/azure/templates/galaxy_script.yml b/tests/azure/templates/galaxy_script.yml index bf14238c..6d20bbd6 100644 --- a/tests/azure/templates/galaxy_script.yml +++ b/tests/azure/templates/galaxy_script.yml @@ -50,6 +50,10 @@ jobs: env: ANSIBLE_LIBRARY: ./molecule + - script: | + python utils/check_test_configuration.py ${{ parameters.scenario }} + displayName: Check scenario test configuration + - script: | cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa pytest \ diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml index abbdf8c0..4001a647 100644 --- a/tests/azure/templates/playbook_tests.yml +++ b/tests/azure/templates/playbook_tests.yml @@ -52,6 +52,10 @@ jobs: env: ANSIBLE_LIBRARY: ./molecule + - script: | + python utils/check_test_configuration.py ${{ parameters.scenario }} + displayName: Check scenario test configuration + - script: | pytest \ -m "playbook" \ diff --git a/tests/sanity/ignore-2.12.txt b/tests/sanity/ignore-2.12.txt index ee85fb0a..73afcd39 100644 --- a/tests/sanity/ignore-2.12.txt +++ b/tests/sanity/ignore-2.12.txt @@ -50,6 +50,7 @@ utils/ansible-ipa-server-install shebang!skip utils/build-galaxy-release.sh shebang!skip utils/build-srpm.sh shebang!skip utils/changelog shebang!skip +utils/check_test_configuration.py shebang!skip utils/galaxyfy-README.py shebang!skip utils/galaxyfy-module-EXAMPLES.py shebang!skip utils/galaxyfy-playbook.py shebang!skip diff --git a/utils/check_test_configuration.py b/utils/check_test_configuration.py new file mode 100755 index 00000000..1500e1dc --- /dev/null +++ b/utils/check_test_configuration.py @@ -0,0 +1,110 @@ +#!/usr/bin/python + +"""Check which tests are scheduled to be executed.""" + +import sys +import re +import os +import yaml + + +RE_IS_TEST = re.compile(r"(.*/)?test_.*\.yml") +RE_IS_VARS = re.compile(r"(.*/)?variables(_.*)?\.yaml") +REPO_ROOT = os.path.join(os.path.dirname(__file__), "..") + + +def get_tests(): + """Retrieve a list of modules and its tests.""" + def get_module(root): + if root != _test_dir: + while True: + module = os.path.basename(root) + root = os.path.dirname(root) + if root == _test_dir: + return module + return "." + + _result = {} + _test_dir = os.path.join(REPO_ROOT, "tests") + for root, _dirs, files in os.walk(_test_dir): + module = get_module(root) + _result[module] = [ + os.path.splitext(test)[0] + for test in files + if RE_IS_TEST.search(test) + ] + + return _result + + +def get_test_config(scenarios): + template_path = os.path.join(REPO_ROOT, "tests/azure/templates") + _result = {} + for _root, _dirs, files in os.walk(template_path): + for filename in [x for x in files if RE_IS_VARS.search(x)]: + _templates, *scenario = os.path.basename( + os.path.splitext(filename)[0] + ).split("_", 1) + scenario = scenario[0] if scenario else "All" + _result[scenario] = {} + # only process selected scenarios + if scenario not in scenarios and len(scenarios) > 1: + continue + with open(os.path.join(template_path, filename), "rt") as inp: + data = yaml.safe_load(inp) + if not data["variables"].get("empty", False): + variables = data["variables"] + for key, value in variables.items(): + variables[key] = [ + x.strip() for x in value.split(",") if x.strip() + ] + _result[scenario] = variables + + return _result + + +def print_configuration(scenario, disabled, enabled): + """Print the test configuration for a scenario.""" + print(f"\nScenario: {scenario}") + for test_cfg, title in [(disabled, "Disabled"), (enabled, "Enabled")]: + print(f" {title} tests:") + if test_cfg: + for module, tests in test_cfg.items(): + print(f" {module}:") + for test in tests: + print(f" - {test}") + else: + print(" No custom configuration.") + + +def main(): + if any(item in sys.argv for item in ["-h", "--help"]): + print("usage: check_test_config.py [-h|--help] [SCENARIO...]") + return + + scenarios = ["All"] + sys.argv[1:] + all_tests = get_tests() + test_config = get_test_config(scenarios) + + print("Test configuration:") + for scenario in sorted(test_config.keys()): + if scenario not in scenarios and len(scenarios) > 1: + continue + # extract scenario configuration + config = test_config[scenario] + disabled = {} + enabled = {} + for res, state in [(disabled, "disabled"), (enabled, "enabled")]: + for module in config.get(f"ipa_{state}_modules", []): + res[module] = set(all_tests[module]) + for test in config.get(f"ipa_{state}_tests", []): + for module, tests in all_tests.items(): + if test in tests: + mod = res.setdefault(module, set()) + mod.add(test) + tests.remove(test) + print_configuration(scenario, disabled, enabled) + + +if __name__ == "__main__": + main()