Allow to run tests in Docker

* Adapted tests/test_playbook_runs.py script to allow tests to be
  executed from a docker container.
* Added molecule scenarios to create/destroy test containers and
  respective documentation in tests/README.md.

tests/README.md

@@ -29,7 +29,13 @@ environment variable. For example:
 ANSIBLE_REMOTE_USER=root IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
 ```
 
-To select which tests to run use the option `-k`. For example:
+To run a single test use the full path with the following format:
+
+```
+IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest tests/test_playbook_runs.py::sudorule::test_sudorule
+```
+
+To select which tests to run based on search use the option `-k`. For example:
 
 ```
 IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -k dnszone
@@ -50,6 +56,45 @@ IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -rs
 For a complete list of options check `pytest --help`.
 
 
+## Running tests in a docker container
+
+It's also possible to run the tests in a container.
+
+### Creating a container to run the tests
+
+Before setting up a container you will need to install molecule framework:
+
+```
+pip install molecule[docker]>=3
+```
+
+Now you can start a test container using the following command:
+```
+molecule create -s centos-8
+```
+
+Note: Currently the containers available for running the tests are:
+ * centos-7
+ * centos-8
+
+### Running the tests inside the container
+
+To run the tests you will use pytest (works the same as for VMs).
+
+```
+RUN_TESTS_IN_DOCKER=1 IPA_SERVER_HOST=centos-8 pytest
+```
+
+### Cleaning up after tests
+
+After running the tests you should probably destroy the test container using:
+
+```
+molecule destroy -s centos-8
+```
+
+See [Running the tests](#running-the-tests) section for more information on available options.
+
 ## Upcoming/desired improvements:
 
 * A script to pre-config the complete test environment using virsh.

tests/test_playbook_runs.py

@@ -4,7 +4,7 @@ import os
 import functools
 import tempfile
 
-from subprocess import Popen
+import subprocess
 
 from unittest import TestCase
 
@@ -13,12 +13,53 @@ import pytest
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+def is_docker_env():
+    if os.getenv("RUN_TESTS_IN_DOCKER", "0") == "0":
+        return False
+    return True
+
+
+def get_server_host():
+    return os.getenv("IPA_SERVER_HOST")
+
+
+def get_molecule_scenario():
+    return get_server_host()
+
+
 def get_inventory_content():
-    ipa_server_host = os.getenv("IPA_SERVER_HOST")
-    return "[ipaserver]\n{}".format(ipa_server_host).encode("utf8")
+    ipa_server_host = get_server_host()
+
+    if is_docker_env():
+        ipa_server_host += " ansible_connection=docker"
+
+    lines = [
+        "[ipaserver]",
+        ipa_server_host,
+        "[ipaserver:vars]",
+        "ipaserver_domain=test.local",
+        "ipaserver_realm=TEST.LOCAL",
+    ]
+    return "\n".join(lines).encode("utf8")
 
 
-def run_playbook(playbook):
+def write_logs(result, test_name):
+    log_dir = os.path.join(SCRIPT_DIR, "logs")
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+
+    # Write stdout log for test
+    log_path = os.path.join(log_dir, test_name + ".log")
+    with open(log_path, "w") as log_file:
+        log_file.write(result.stdout.decode("utf-8"))
+
+    # Write stderr log for test
+    error_log_path = os.path.join(log_dir, test_name + "-error.log")
+    with open(error_log_path, "w") as log_file:
+        log_file.write(result.stderr.decode("utf-8"))
+
+
+def run_playbook(playbook, test_name):
     with tempfile.NamedTemporaryFile() as inventory_file:
         inventory_file.write(get_inventory_content())
         inventory_file.flush()
@@ -28,15 +69,17 @@ def run_playbook(playbook):
             inventory_file.name,
             playbook,
         ]
-        process = Popen(cmd, cwd=SCRIPT_DIR)
-        process.wait()
+        process = subprocess.run(
+            cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+    write_logs(process, test_name)
 
     return process
 
 
 def list_test_yaml(dir_path):
     yamls = []
-    for yaml_name in os.listdir(dir_path):
+    for yaml_name in sorted(os.listdir(dir_path)):
         if yaml_name.startswith("test_") and yaml_name.endswith(".yml"):
             yamls.append(
                 {
@@ -50,7 +93,7 @@ def list_test_yaml(dir_path):
 def get_test_groups():
     test_dirs = os.listdir(SCRIPT_DIR)
     groups = {}
-    for test_group_dir in test_dirs:
+    for test_group_dir in sorted(test_dirs):
         group_dir_path = os.path.join(SCRIPT_DIR, test_group_dir)
         if not os.path.isdir(group_dir_path):
             continue
@@ -65,6 +108,7 @@ def prepare_test(test_name, test_path):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
             kwargs["test_path"] = test_path
+            kwargs["test_name"] = test_name
             return func(*args, **kwargs)
 
         return wrapper
@@ -82,13 +126,29 @@ for group_name, group_tests in get_test_groups().items():
         test_path = test_config["path"]
 
         @pytest.mark.skipif(
-            os.getenv("IPA_SERVER_HOST") is None,
+            not get_server_host(),
             reason="Environment variable IPA_SERVER_HOST must be set",
         )
         @prepare_test(test_name, test_path)
-        def method(self, test_path):
-            result = run_playbook(test_path)
-            assert result.returncode == 0
+        def method(self, test_path, test_name):
+            result = run_playbook(test_path, test_name)
+            status_code_msg = "ansible-playbook return code: {}".format(
+                result.returncode
+            )
+            assert_msg = "\n".join(
+                [
+                    "",
+                    "-" * 30 + " Captured stdout " + "-" * 30,
+                    result.stdout.decode("utf8"),
+                    "-" * 30 + " Captured stderr " + "-" * 30,
+                    result.stderr.decode("utf8"),
+                    "-" * 30 + " Playbook Return Code " + "-" * 30,
+                    status_code_msg,
+                ]
+            )
+            # Need to get the last bytes of msg otherwise Azure
+            #   will cut it out.
+            assert result.returncode == 0, assert_msg[-2500:]
 
         _tests[test_name] = method
-    globals()[group_name] = type(group_name, (TestCase,), _tests)
+    globals()[group_name] = type(group_name, tuple([TestCase]), _tests,)