Merge pull request #884 from rjeffman/ci_enable_distro_selection

upstream CI: enable/disable tests based on test image

tests/azure/templates/galaxy_script.yml

@@ -25,6 +25,7 @@ jobs:
   timeoutInMinutes: 120
   variables:
   - template: variables.yaml
+  - template: variables_${{ parameters.scenario }}.yaml
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -49,6 +50,10 @@ jobs:
     env:
       ANSIBLE_LIBRARY: ./molecule
 
+  - script: |
+      python utils/check_test_configuration.py ${{ parameters.scenario }}
+    displayName: Check scenario test configuration
+
   - script: |
       cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa
       pytest \

tests/azure/templates/playbook_tests.yml

@@ -24,6 +24,7 @@ jobs:
   timeoutInMinutes: 120
   variables:
   - template: variables.yaml
+  - template: variables_${{ parameters.scenario }}.yaml
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -51,6 +52,10 @@ jobs:
     env:
       ANSIBLE_LIBRARY: ./molecule
 
+  - script: |
+      python utils/check_test_configuration.py ${{ parameters.scenario }}
+    displayName: Check scenario test configuration
+
   - script: |
       pytest \
         -m "playbook" \

tests/azure/templates/pytest_tests.yml

@@ -18,6 +18,7 @@ jobs:
   timeoutInMinutes: 120
   variables:
   - template: variables.yaml
+  - template: variables_${{ parameters.scenario }}.yaml
   steps:
   - task: UsePythonVersion@0
     inputs:

tests/azure/templates/variables.yaml

@@ -3,6 +3,9 @@
 # For easier management of items to enable/disable,
 # use one test/module on each line, followed by a comma.
 #
+# If no variable is to be set, add 'empty: true', as the
+# 'variables' dict cannot be empty.
+#
 # Example:
 #
 # disabled_modules: >-
@@ -12,9 +15,8 @@
 #
 ---
 variables:
+  empty: true
   # ipa_enabled_modules: >-
   # ipa_enabled_tests: >-
-  ipa_disabled_modules: >-
-    dnsconfig,
-  ipa_disabled_tests: >-
-    test_dnsconfig_forwarders_ports
+  # ipa_disabled_modules: >-
+  # ipa_disabled_tests: >-

tests/azure/templates/variables_c8s.yaml

@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+---
+variables:
+  empty: true
+#   ipa_enabled_modules: >-
+#   ipa_enabled_tests: >-
+#   ipa_disabled_modules: >-
+#   ipa_disabled_tests: >-

tests/azure/templates/variables_c9s.yaml

@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+---
+variables:
+  empty: true
+#   ipa_enabled_modules: >-
+#   ipa_enabled_tests: >-
+#   ipa_disabled_modules: >-
+#   ipa_disabled_tests: >-

tests/azure/templates/variables_centos-7.yaml

@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+---
+variables:
+  empty: true
+#   ipa_enabled_modules: >-
+#   ipa_enabled_tests: >-
+#   ipa_disabled_modules: >-
+#   ipa_disabled_tests: >-

tests/azure/templates/variables_fedora-latest.yaml

@@ -0,0 +1,20 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+---
+variables:
+  # ipa_enabled_modules: >-
+  # ipa_enabled_tests: >-
+  ipa_disabled_modules: >-
+    dnsforwardzone,
+  ipa_disabled_tests: >-
+    test_dnsconfig_forwarders_ports

tests/azure/templates/variables_fedora-rawhide.yaml

@@ -0,0 +1,19 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+---
+variables:
+  empty: true
+  # ipa_enabled_modules: >-
+  # ipa_enabled_tests: >-
+  # ipa_disabled_modules: >-
+  # ipa_disabled_tests: >-

tests/sanity/ignore-2.12.txt

@@ -50,6 +50,7 @@ utils/ansible-ipa-server-install shebang!skip
 utils/build-galaxy-release.sh shebang!skip
 utils/build-srpm.sh shebang!skip
 utils/changelog shebang!skip
+utils/check_test_configuration.py shebang!skip
 utils/galaxyfy-README.py shebang!skip
 utils/galaxyfy-module-EXAMPLES.py shebang!skip
 utils/galaxyfy-playbook.py shebang!skip

tests/test_playbook_runs.py

@@ -24,7 +24,10 @@ import functools
 
 from unittest import TestCase
 
-from utils import get_test_playbooks, get_skip_conditions, run_playbook
+from utils import (
+    get_test_playbooks, get_server_host, run_playbook, get_enabled_test,
+    get_disabled_test
+)
 
 
 def prepare_test(testname, testpath):
@@ -47,6 +50,9 @@ def prepare_test(testname, testpath):
     return decorator
 
 
+if not get_server_host():
+    raise RuntimeError("IPA_SERVER_HOST is not set.")
+
 # Dynamically create the TestCase classes with respective
 #   test_* methods.
 for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
@@ -56,16 +62,17 @@ for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
         test_name = playbook["name"].replace("-", "_")
         test_path = playbook["path"]
 
-        skip = get_skip_conditions(test_dir_name, test_name) or {}
+        if (
+            get_enabled_test(test_dir_name, test_name)
+            and not get_disabled_test(test_dir_name, test_name)
+        ):
+            # pylint: disable=W0621,W0640,W0613
+            @pytest.mark.playbook
+            @prepare_test(test_name, test_path)
+            def method(self, test_path, test_name):
+                run_playbook(test_path)
+            # pylint: enable=W0621,W0640,W0613
 
-        # pylint: disable=W0621,W0640,W0613
-        @pytest.mark.skipif(**skip)
-        @pytest.mark.playbook
-        @prepare_test(test_name, test_path)
-        def method(self, test_path, test_name):
-            run_playbook(test_path)
-        # pylint: enable=W0621,W0640,W0613
-
-        _tests[test_name] = method
+            _tests[test_name] = method
 
     globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,)

tests/utils.py

@@ -84,30 +84,6 @@ def get_enabled_test(group_name, test_name):
     return group_enabled or test_enabled
 
 
-def get_skip_conditions(group_name, test_name):
-    """
-    Check tests that need to be skipped.
-
-    The return is a dict containing `condition` and `reason`. For the test
-    to be skipped, `condition` must be True, if it is `False`, the test is
-    to be skipped. Although "reason" must be always provided, it can be
-    `None` if `condition` is True.
-    """
-    if not get_server_host():
-        return {
-            "condition": True,
-            "reason": "Environment variable IPA_SERVER_HOST must be set",
-        }
-
-    if not get_enabled_test(group_name, test_name):
-        return {"condition": True, "reason": "Test not configured to run"}
-
-    if get_disabled_test(group_name, test_name):
-        return {"condition": True, "reason": "Test configured to not run"}
-
-    return {"condition": False, "reason": "Test will run."}
-
-
 def get_inventory_content():
     """Create the content of an inventory file for a test run."""
     ipa_server_host = get_server_host()

utils/check_test_configuration.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python
+
+"""Check which tests are scheduled to be executed."""
+
+import sys
+import re
+import os
+import yaml
+
+
+RE_IS_TEST = re.compile(r"(.*/)?test_.*\.yml")
+RE_IS_VARS = re.compile(r"(.*/)?variables(_.*)?\.yaml")
+REPO_ROOT = os.path.join(os.path.dirname(__file__), "..")
+
+
+def get_tests():
+    """Retrieve a list of modules and its tests."""
+    def get_module(root):
+        if root != _test_dir:
+            while True:
+                module = os.path.basename(root)
+                root = os.path.dirname(root)
+                if root == _test_dir:
+                    return module
+        return "."
+
+    _result = {}
+    _test_dir = os.path.join(REPO_ROOT, "tests")
+    for root, _dirs, files in os.walk(_test_dir):
+        module = get_module(root)
+        _result[module] = [
+            os.path.splitext(test)[0]
+            for test in files
+            if RE_IS_TEST.search(test)
+        ]
+
+    return _result
+
+
+def get_test_config(scenarios):
+    template_path = os.path.join(REPO_ROOT, "tests/azure/templates")
+    _result = {}
+    for _root, _dirs, files in os.walk(template_path):
+        for filename in [x for x in files if RE_IS_VARS.search(x)]:
+            _templates, *scenario = os.path.basename(
+                os.path.splitext(filename)[0]
+            ).split("_", 1)
+            scenario = scenario[0] if scenario else "All"
+            _result[scenario] = {}
+            # only process selected scenarios
+            if scenario not in scenarios and len(scenarios) > 1:
+                continue
+            with open(os.path.join(template_path, filename), "rt") as inp:
+                data = yaml.safe_load(inp)
+            if not data["variables"].get("empty", False):
+                variables = data["variables"]
+                for key, value in variables.items():
+                    variables[key] = [
+                        x.strip() for x in value.split(",") if x.strip()
+                    ]
+                _result[scenario] = variables
+
+    return _result
+
+
+def print_configuration(scenario, disabled, enabled):
+    """Print the test configuration for a scenario."""
+    print(f"\nScenario: {scenario}")
+    for test_cfg, title in [(disabled, "Disabled"), (enabled, "Enabled")]:
+        print(f"    {title} tests:")
+        if test_cfg:
+            for module, tests in test_cfg.items():
+                print(f"        {module}:")
+                for test in tests:
+                    print(f"            - {test}")
+        else:
+            print("        No custom configuration.")
+
+
+def main():
+    if any(item in sys.argv for item in ["-h", "--help"]):
+        print("usage: check_test_config.py [-h|--help] [SCENARIO...]")
+        return
+
+    scenarios = ["All"] + sys.argv[1:]
+    all_tests = get_tests()
+    test_config = get_test_config(scenarios)
+
+    print("Test configuration:")
+    for scenario in sorted(test_config.keys()):
+        if scenario not in scenarios and len(scenarios) > 1:
+            continue
+        # extract scenario configuration
+        config = test_config[scenario]
+        disabled = {}
+        enabled = {}
+        for res, state in [(disabled, "disabled"), (enabled, "enabled")]:
+            for module in config.get(f"ipa_{state}_modules", []):
+                res[module] = set(all_tests[module])
+            for test in config.get(f"ipa_{state}_tests", []):
+                for module, tests in all_tests.items():
+                    if test in tests:
+                        mod = res.setdefault(module, set())
+                        mod.add(test)
+                        tests.remove(test)
+        print_configuration(scenario, disabled, enabled)
+
+
+if __name__ == "__main__":
+    main()