upstream ci: Add step to display scenario configuration

Since test configuration can vary in different scenarios (test images)
this patch adds a script to list the scenarios configuration, and a
step to the playbook test jobs to display the scenario configuration.

tests/azure/templates/galaxy_script.yml

@@ -50,6 +50,10 @@ jobs:
     env:
       ANSIBLE_LIBRARY: ./molecule
 
+  - script: |
+      python utils/check_test_configuration.py ${{ parameters.scenario }}
+    displayName: Check scenario test configuration
+
   - script: |
       cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa
       pytest \

tests/azure/templates/playbook_tests.yml

@@ -52,6 +52,10 @@ jobs:
     env:
       ANSIBLE_LIBRARY: ./molecule
 
+  - script: |
+      python utils/check_test_configuration.py ${{ parameters.scenario }}
+    displayName: Check scenario test configuration
+
   - script: |
       pytest \
         -m "playbook" \

tests/sanity/ignore-2.12.txt

@@ -50,6 +50,7 @@ utils/ansible-ipa-server-install shebang!skip
 utils/build-galaxy-release.sh shebang!skip
 utils/build-srpm.sh shebang!skip
 utils/changelog shebang!skip
+utils/check_test_configuration.py shebang!skip
 utils/galaxyfy-README.py shebang!skip
 utils/galaxyfy-module-EXAMPLES.py shebang!skip
 utils/galaxyfy-playbook.py shebang!skip

utils/check_test_configuration.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python
+
+"""Check which tests are scheduled to be executed."""
+
+import sys
+import re
+import os
+import yaml
+
+
+RE_IS_TEST = re.compile(r"(.*/)?test_.*\.yml")
+RE_IS_VARS = re.compile(r"(.*/)?variables(_.*)?\.yaml")
+REPO_ROOT = os.path.join(os.path.dirname(__file__), "..")
+
+
+def get_tests():
+    """Retrieve a list of modules and its tests."""
+    def get_module(root):
+        if root != _test_dir:
+            while True:
+                module = os.path.basename(root)
+                root = os.path.dirname(root)
+                if root == _test_dir:
+                    return module
+        return "."
+
+    _result = {}
+    _test_dir = os.path.join(REPO_ROOT, "tests")
+    for root, _dirs, files in os.walk(_test_dir):
+        module = get_module(root)
+        _result[module] = [
+            os.path.splitext(test)[0]
+            for test in files
+            if RE_IS_TEST.search(test)
+        ]
+
+    return _result
+
+
+def get_test_config(scenarios):
+    template_path = os.path.join(REPO_ROOT, "tests/azure/templates")
+    _result = {}
+    for _root, _dirs, files in os.walk(template_path):
+        for filename in [x for x in files if RE_IS_VARS.search(x)]:
+            _templates, *scenario = os.path.basename(
+                os.path.splitext(filename)[0]
+            ).split("_", 1)
+            scenario = scenario[0] if scenario else "All"
+            _result[scenario] = {}
+            # only process selected scenarios
+            if scenario not in scenarios and len(scenarios) > 1:
+                continue
+            with open(os.path.join(template_path, filename), "rt") as inp:
+                data = yaml.safe_load(inp)
+            if not data["variables"].get("empty", False):
+                variables = data["variables"]
+                for key, value in variables.items():
+                    variables[key] = [
+                        x.strip() for x in value.split(",") if x.strip()
+                    ]
+                _result[scenario] = variables
+
+    return _result
+
+
+def print_configuration(scenario, disabled, enabled):
+    """Print the test configuration for a scenario."""
+    print(f"\nScenario: {scenario}")
+    for test_cfg, title in [(disabled, "Disabled"), (enabled, "Enabled")]:
+        print(f"    {title} tests:")
+        if test_cfg:
+            for module, tests in test_cfg.items():
+                print(f"        {module}:")
+                for test in tests:
+                    print(f"            - {test}")
+        else:
+            print("        No custom configuration.")
+
+
+def main():
+    if any(item in sys.argv for item in ["-h", "--help"]):
+        print("usage: check_test_config.py [-h|--help] [SCENARIO...]")
+        return
+
+    scenarios = ["All"] + sys.argv[1:]
+    all_tests = get_tests()
+    test_config = get_test_config(scenarios)
+
+    print("Test configuration:")
+    for scenario in sorted(test_config.keys()):
+        if scenario not in scenarios and len(scenarios) > 1:
+            continue
+        # extract scenario configuration
+        config = test_config[scenario]
+        disabled = {}
+        enabled = {}
+        for res, state in [(disabled, "disabled"), (enabled, "enabled")]:
+            for module in config.get(f"ipa_{state}_modules", []):
+                res[module] = set(all_tests[module])
+            for test in config.get(f"ipa_{state}_tests", []):
+                for module, tests in all_tests.items():
+                    if test in tests:
+                        mod = res.setdefault(module, set())
+                        mod.add(test)
+                        tests.remove(test)
+        print_configuration(scenario, disabled, enabled)
+
+
+if __name__ == "__main__":
+    main()