Merge pull request #849 from rjeffman/dev_run_tests_locally

Run tests locally with upstream CI images.

tests/README.md

@@ -138,6 +138,35 @@ molecule destroy -s c8s
 See [Running the tests](#running-the-tests) section for more information on available options.
 
 
+## Running local tests with upstream CI images
+
+To run tests locally using the same images used by upstream CI use `utils/run-tests.sh`.
+
+```
+utils/run-tests.sh tests/config/test_config.yml
+```
+
+To run all tests for a single plugin, use the `-s` option with plugin directory name. This will search, recursively for playbooks named with the pattern `test_*.yml`. To run all playbook tests for `ipauser`:
+
+```
+utils/run-tests.sh -s user
+```
+
+When executed, `utils/run-tests.sh` will create a container (either using `docker` or `podman`) using one of the testing ansible-freeipa images (https://quay.io/repository/ansible-freeipa/upstream-tests?tab=tags), run the selected tests against the container, and remove the container after tests are executed. If a test fails the container is not removed, so the failure can be investigated.
+
+It is possible to keep the container after the execution, even if tests succeed, using the `-C` option:
+
+```
+utils/run-tests.sh -s config -C
+```
+
+By default the tests are executed against the latest version of the Fedora image (`fedora-latest`). The testing image can be selected with the `-i` option. Use `-l` to list the available image names.
+
+```
+utils/run-tests.sh -i c9s tests/host/test_host.yml
+```
+
+
 ## Upcoming/desired improvements:
 
 * A script to pre-config the complete test environment using virsh.

tests/utils.py

@@ -32,10 +32,11 @@ from unittest import TestCase
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def is_docker_env():
-    if os.getenv("RUN_TESTS_IN_DOCKER", "0") == "0":
-        return False
-    return True
+def get_docker_env():
+    docker_env = os.getenv("RUN_TESTS_IN_DOCKER", None)
+    if docker_env in ["1", "True", "true", "yes", True]:
+        docker_env = "docker"
+    return docker_env
 
 
 def get_ssh_password():
@@ -88,8 +89,9 @@ def get_inventory_content():
     """Create the content of an inventory file for a test run."""
     ipa_server_host = get_server_host()
 
-    if is_docker_env():
-        ipa_server_host += " ansible_connection=docker"
+    container_engine = get_docker_env()
+    if container_engine is not None:
+        ipa_server_host += f" ansible_connection={container_engine}"
 
     sshpass = get_ssh_password()
     if sshpass:
@@ -145,12 +147,11 @@ def _run_playbook(playbook):
     with tempfile.NamedTemporaryFile() as inventory_file:
         inventory_file.write(get_inventory_content())
         inventory_file.flush()
-        cmd = [
-            "ansible-playbook",
-            "-i",
-            inventory_file.name,
-            playbook,
-        ]
+        cmd_options = ["-i", inventory_file.name]
+        verbose = os.environ.get("IPA_VERBOSITY", None)
+        if verbose is not None:
+            cmd_options.append(verbose)
+        cmd = ["ansible-playbook"] + cmd_options + [playbook]
         # pylint: disable=subprocess-run-check
         process = subprocess.run(
             cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
@@ -257,8 +258,9 @@ def kdestroy(host):
 
 class AnsibleFreeIPATestCase(TestCase):
     def setUp(self):
-        if is_docker_env():
-            protocol = "docker://"
+        container_engine = get_docker_env()
+        if container_engine:
+            protocol = f"{container_engine}://"
             user = ""
             ssh_identity_file = None
         else: