Merge pull request #354 from rjeffman/tests_ignore_tests

Add support to define which playbook tests to execute with pytest.

tests/README.md

@@ -63,6 +63,24 @@ IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -rs
 
 For a complete list of options check `pytest --help`.
 
+### Disabling and enabling playbook tests
+
+Sometimes it is useful to enable or disable specific playbook tests. To only run a subset of modules or tests, use the variables IPA_ENABLED_MODULES and IPA ENABLED_TESTS, to define a comma-separated list of modules or tests to be enabled. Any test or module not in the list will not be executed. For example, to run only `sudorule` and `sudocmd` tests:
+
+```
+IPA_ENABLE_MODULES="sudorule,sudocmd" IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
+```
+
+If all but a few selected tests are to be executed, use the IPA_DISABLED_MODULES or IPA_DISABLED_TESTS. For example, to run all, but "test_service_certificate" test:
+
+```
+IPA_DISABLED_TESTS=test_service_certificate IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
+```
+
+If none of this variables are defined, all tests will be executed.
+
+To configure the tests that will run for your pull request, add a TEMP commit, with the configuration defined in the file `tests/azure/templates/variables.yml`. Set the variables `ipa_enable_modules`, `ipa_enable_tests`, `ipa_disable_modules`, and `ipa_disable_tests`, in the same way as the equivalent environment variables.
+
 ### Types of tests
 
 #### Playbook tests
@@ -119,6 +137,7 @@ molecule destroy -s c8s
 
 See [Running the tests](#running-the-tests) section for more information on available options.
 
+
 ## Upcoming/desired improvements:
 
 * A script to pre-config the complete test environment using virsh.

tests/azure/templates/playbook_tests.yml

@@ -18,11 +18,12 @@ parameters:
   - name: build_number
     type: string
 
-
 jobs:
 - job: Test_Group${{ parameters.group_number }}
   displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
   timeoutInMinutes: 120
+  variables:
+  - template: variables.yaml
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -63,6 +64,10 @@ jobs:
     env:
       IPA_SERVER_HOST: ${{ parameters.scenario }}
       RUN_TESTS_IN_DOCKER: true
+      IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
+      IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+      IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
+      IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
 
   - task: PublishTestResults@2
     inputs:

tests/azure/templates/pytest_tests.yml

@@ -16,6 +16,8 @@ jobs:
 - job: Test_PyTests
   displayName: Run pytests on ${{ parameters.scenario }}
   timeoutInMinutes: 120
+  variables:
+  - template: variables.yaml
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -53,6 +55,10 @@ jobs:
     env:
       IPA_SERVER_HOST: ${{ parameters.scenario }}
       RUN_TESTS_IN_DOCKER: true
+      IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
+      IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
+      IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }}
+      IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }}
 
   - task: PublishTestResults@2
     inputs:

tests/azure/templates/variables.yaml

@@ -0,0 +1,20 @@
+#
+# Variables must be defined as comma separated lists.
+# For easier management of items to enable/disable,
+# use one test/module on each line, followed by a comma.
+#
+# Example:
+#
+# disabled_modules: >-
+#   dnsconfig,
+#   group,
+#   hostgroup
+#
+---
+variables:
+  # ipa_enabled_modules: >-
+  # ipa_enabled_tests: >-
+  ipa_disabled_modules: >-
+    dnsconfig,
+    dnsforwardzone,
+  # ipa_disabled_tests: >-

tests/sanity/ignore-2.12.txt

@@ -43,7 +43,6 @@ tests/sanity/sanity.sh shebang!skip
 tests/user/users.sh shebang!skip
 tests/user/users_absent.sh shebang!skip
 tests/utils.py pylint:ansible-format-automatic-specification
-tests/utils.py pylint:subprocess-run-check
 utils/ansible-doc-test shebang!skip
 utils/ansible-ipa-client-install shebang!skip
 utils/ansible-ipa-replica-install shebang!skip

tests/test_playbook_runs.py

@@ -24,11 +24,12 @@ import functools
 
 from unittest import TestCase
 
-from utils import get_test_playbooks, get_server_host, run_playbook
+from utils import get_test_playbooks, get_skip_conditions, run_playbook
 
 
-def prepare_test(test_name, test_path):
-    """Decorator for the tests generated automatically from playbooks.
+def prepare_test(testname, testpath):
+    """
+    Decorate tests generated automatically from playbooks.
 
     Injects 2 arguments to the test (`test_path` and `test_name`) and
     name the test method using test name (to ensure test reports are useful).
@@ -36,13 +37,13 @@ def prepare_test(test_name, test_path):
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
-            kwargs["test_path"] = test_path
-            kwargs["test_name"] = test_name
+            kwargs["test_path"] = testpath
+            kwargs["test_name"] = testname
             return func(*args, **kwargs)
 
         return wrapper
 
-    decorator.__name__ = test_name
+    decorator.__name__ = testname
     return decorator
 
 
@@ -50,18 +51,21 @@ def prepare_test(test_name, test_path):
 #   test_* methods.
 for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
     _tests = {}
+
     for playbook in playbooks_in_dir:
         test_name = playbook["name"].replace("-", "_")
         test_path = playbook["path"]
 
-        @pytest.mark.skipif(
-            not get_server_host(),
-            reason="Environment variable IPA_SERVER_HOST must be set",
-        )
+        skip = get_skip_conditions(test_dir_name, test_name) or {}
+
+        # pylint: disable=W0621,W0640,W0613
+        @pytest.mark.skipif(**skip)
         @pytest.mark.playbook
         @prepare_test(test_name, test_path)
         def method(self, test_path, test_name):
             run_playbook(test_path)
+        # pylint: enable=W0621,W0640,W0613
 
         _tests[test_name] = method
+
     globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,)

tests/utils.py

@@ -45,6 +45,68 @@ def get_server_host():
     return os.getenv("IPA_SERVER_HOST")
 
 
+def get_disabled_test(group_name, test_name):
+    disabled_modules = [
+        disabled.strip()
+        for disabled in os.environ.get("IPA_DISABLED_MODULES", "").split(",")
+    ]
+    disabled_tests = [
+        disabled.strip()
+        for disabled in os.environ.get("IPA_DISABLED_TESTS", "").split(",")
+        if disabled.strip()
+    ]
+
+    if not any([disabled_modules, disabled_tests]):
+        return False
+
+    return group_name in disabled_modules or test_name in disabled_tests
+
+
+def get_enabled_test(group_name, test_name):
+    enabled_modules = [
+        enabled.strip()
+        for enabled in os.environ.get("IPA_ENABLED_MODULES", "").split(":")
+        if enabled.strip()
+    ]
+    enabled_tests = [
+        enabled.strip()
+        for enabled in os.environ.get("IPA_ENABLED_TESTS", "").split(":")
+        if enabled.strip()
+    ]
+
+    if not any([enabled_modules, enabled_tests]):
+        return True
+
+    group_enabled = group_name in enabled_modules
+    test_enabled = test_name in enabled_tests
+
+    return group_enabled or test_enabled
+
+
+def get_skip_conditions(group_name, test_name):
+    """
+    Check tests that need to be skipped.
+
+    The return is a dict containing `condition` and `reason`. For the test
+    to be skipped, `condition` must be True, if it is `False`, the test is
+    to be skipped. Although "reason" must be always provided, it can be
+    `None` if `condition` is True.
+    """
+    if not get_server_host():
+        return {
+            "condition": True,
+            "reason": "Environment variable IPA_SERVER_HOST must be set",
+        }
+
+    if not get_enabled_test(group_name, test_name):
+        return {"condition": True, "reason": "Test not configured to run"}
+
+    if get_disabled_test(group_name, test_name):
+        return {"condition": True, "reason": "Test configured to not run"}
+
+    return {"condition": False, "reason": "Test will run."}
+
+
 def get_inventory_content():
     """Create the content of an inventory file for a test run."""
     ipa_server_host = get_server_host()
@@ -112,6 +174,7 @@ def _run_playbook(playbook):
             inventory_file.name,
             playbook,
         ]
+        # pylint: disable=subprocess-run-check
         process = subprocess.run(
             cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
         )
@@ -238,11 +301,13 @@ class AnsibleFreeIPATestCase(TestCase):
             host_connection_info, ssh_identity_file=ssh_identity_file,
         )
 
-    def run_playbook(self, playbook, allow_failures=False):
+    @staticmethod
+    def run_playbook(playbook, allow_failures=False):
         return run_playbook(playbook, allow_failures)
 
-    def run_playbook_with_exp_msg(self, playbook, expected_msg):
-        result = self.run_playbook(playbook, allow_failures=True)
+    @staticmethod
+    def run_playbook_with_exp_msg(playbook, expected_msg):
+        result = run_playbook(playbook, allow_failures=True)
         assert (
             expected_msg in result.stdout.decode("utf8")
             or