Allow to run tests in Docker

* Adapted tests/test_playbook_runs.py script to allow tests to be
  executed from a docker container.
* Added molecule scenarios to create/destroy test containers and
  respective documentation in tests/README.md.

molecule/centos-7-build/molecule.yml

@@ -0,0 +1,20 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: centos-7-build
+    image: centos/systemd
+    pre_build_image: true
+    hostname: ipaserver.test.local
+    dns_servers:
+      - 8.8.8.8
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    command: /usr/sbin/init
+    privileged: true
+provisioner:
+  name: ansible
+  playbooks:
+    prepare: ../resources/playbooks/prepare-build.yml

molecule/centos-7/molecule.yml

@@ -0,0 +1,20 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: centos-7
+    image: quay.io/ansible-freeipa/upstream-tests:centos-7
+    pre_build_image: true
+    hostname: ipaserver.test.local
+    dns_servers:
+      - 127.0.0.1
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    command: /usr/sbin/init
+    privileged: true
+provisioner:
+  name: ansible
+  playbooks:
+    prepare: ../resources/playbooks/prepare.yml

molecule/centos-8-build/molecule.yml

@@ -0,0 +1,20 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: centos-8-build
+    image: centos:8
+    pre_build_image: true
+    hostname: ipaserver.test.local
+    dns_servers:
+      - 8.8.8.8
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    command: /usr/sbin/init
+    privileged: true
+provisioner:
+  name: ansible
+  playbooks:
+    prepare: ../resources/playbooks/prepare-build.yml

molecule/centos-8/molecule.yml

@@ -0,0 +1,20 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: centos-8
+    image: quay.io/ansible-freeipa/upstream-tests:centos-8
+    pre_build_image: true
+    hostname: ipaserver.test.local
+    dns_servers:
+      - 127.0.0.1
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    command: /usr/sbin/init
+    privileged: true
+provisioner:
+  name: ansible
+  playbooks:
+    prepare: ../resources/playbooks/prepare.yml

molecule/default

@@ -0,0 +1 @@
+/home/scampos/src/ansible-freeipa/molecule/centos-8

molecule/resources/playbooks/library

@@ -0,0 +1 @@
+/home/scampos/src/ansible-freeipa/plugins/modules/

molecule/resources/playbooks/module_utils

@@ -0,0 +1 @@
+/home/scampos/src/ansible-freeipa/plugins/module_utils/

molecule/resources/playbooks/prepare-build.yml

@@ -0,0 +1,54 @@
+---
+- name: Converge
+  hosts: all
+  tasks:
+  - name: Ensure IPv6 is ENABLED
+    sysctl:
+      name: "{{ item.name }}"
+      value: "{{ item.value }}"
+      sysctl_set: yes
+      state: present
+      reload: yes
+    with_items :
+      - name: net.ipv6.conf.all.disable_ipv6
+        value: 0
+      - name: net.ipv6.conf.lo.disable_ipv6
+        value: 0
+      - name: net.ipv6.conf.eth0.disable_ipv6
+        value: 1
+
+  - name: stat protected_regular
+    stat:
+      path: /proc/sys/fs/protected_regular
+    register: result
+
+  - name: Ensure fs.protected_regular is disabled
+    sysctl:
+      name: fs.protected_regular
+      value: 0
+      sysctl_set: yes
+      state: present
+      reload: yes
+    when: result.stat.exists
+
+  - name: Ensure sudo package is installed
+    package:
+      name: sudo
+
+  - name: Ensure nss package is updated
+    package:
+      name: nss
+      state: latest
+
+  - include_role:
+      name: ipaserver
+    vars:
+      ipaserver_setup_dns: yes
+      ipaserver_setup_kra: yes
+      ipaserver_auto_forwarders: yes
+      ipaserver_no_dnssec_validation: yes
+      ipaserver_auto_reverse: yes
+      ipaadmin_password: SomeADMINpassword
+      ipadm_password: SomeDMpassword
+      ipaserver_domain: test.local
+      ipaserver_realm: TEST.LOCAL

molecule/resources/playbooks/prepare.yml

@@ -0,0 +1,18 @@
+---
+- name: Converge
+  hosts: all
+  tasks:
+  - name: Ensure lock dirs for DS exists
+    file:
+      state: directory
+      owner: dirsrv
+      group: dirsrv
+      path: "{{ item }} "
+    loop:
+      - /var/lock/dirsrv/
+      - /var/lock/dirsrv/slapd-TEST-LOCAL/
+
+  - name: Ensure IPA server is up an running
+    service:
+      name: ipa
+      state: started

molecule/resources/playbooks/roles

@@ -0,0 +1 @@
+/home/scampos/src/ansible-freeipa/roles

pytest.ini

@@ -1,2 +1,3 @@
 [pytest]
 python_files = test_*.py
+junit_family = xunit1

tests/README.md

@@ -29,7 +29,13 @@ environment variable. For example:
 ANSIBLE_REMOTE_USER=root IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
 ```
 
-To select which tests to run use the option `-k`. For example:
+To run a single test use the full path with the following format:
+
+```
+IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest tests/test_playbook_runs.py::sudorule::test_sudorule
+```
+
+To select which tests to run based on search use the option `-k`. For example:
 
 ```
 IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -k dnszone
@@ -50,6 +56,45 @@ IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -rs
 For a complete list of options check `pytest --help`.
 
 
+## Running tests in a docker container
+
+It's also possible to run the tests in a container.
+
+### Creating a container to run the tests
+
+Before setting up a container you will need to install molecule framework:
+
+```
+pip install molecule[docker]>=3
+```
+
+Now you can start a test container using the following command:
+```
+molecule create -s centos-8
+```
+
+Note: Currently the containers available for running the tests are:
+ * centos-7
+ * centos-8
+
+### Running the tests inside the container
+
+To run the tests you will use pytest (works the same as for VMs).
+
+```
+RUN_TESTS_IN_DOCKER=1 IPA_SERVER_HOST=centos-8 pytest
+```
+
+### Cleaning up after tests
+
+After running the tests you should probably destroy the test container using:
+
+```
+molecule destroy -s centos-8
+```
+
+See [Running the tests](#running-the-tests) section for more information on available options.
+
 ## Upcoming/desired improvements:
 
 * A script to pre-config the complete test environment using virsh.

tests/test_playbook_runs.py

@@ -4,7 +4,7 @@ import os
 import functools
 import tempfile
 
-from subprocess import Popen
+import subprocess
 
 from unittest import TestCase
 
@@ -13,12 +13,53 @@ import pytest
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+def is_docker_env():
+    if os.getenv("RUN_TESTS_IN_DOCKER", "0") == "0":
+        return False
+    return True
+
+
+def get_server_host():
+    return os.getenv("IPA_SERVER_HOST")
+
+
+def get_molecule_scenario():
+    return get_server_host()
+
+
 def get_inventory_content():
-    ipa_server_host = os.getenv("IPA_SERVER_HOST")
+    ipa_server_host = get_server_host()
-    return "[ipaserver]\n{}".format(ipa_server_host).encode("utf8")
+
+    if is_docker_env():
+        ipa_server_host += " ansible_connection=docker"
+
+    lines = [
+        "[ipaserver]",
+        ipa_server_host,
+        "[ipaserver:vars]",
+        "ipaserver_domain=test.local",
+        "ipaserver_realm=TEST.LOCAL",
+    ]
+    return "\n".join(lines).encode("utf8")
 
 
-def run_playbook(playbook):
+def write_logs(result, test_name):
+    log_dir = os.path.join(SCRIPT_DIR, "logs")
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+
+    # Write stdout log for test
+    log_path = os.path.join(log_dir, test_name + ".log")
+    with open(log_path, "w") as log_file:
+        log_file.write(result.stdout.decode("utf-8"))
+
+    # Write stderr log for test
+    error_log_path = os.path.join(log_dir, test_name + "-error.log")
+    with open(error_log_path, "w") as log_file:
+        log_file.write(result.stderr.decode("utf-8"))
+
+
+def run_playbook(playbook, test_name):
     with tempfile.NamedTemporaryFile() as inventory_file:
         inventory_file.write(get_inventory_content())
         inventory_file.flush()
@@ -28,15 +69,17 @@ def run_playbook(playbook):
             inventory_file.name,
             playbook,
         ]
-        process = Popen(cmd, cwd=SCRIPT_DIR)
+        process = subprocess.run(
-        process.wait()
+            cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+    write_logs(process, test_name)
 
     return process
 
 
 def list_test_yaml(dir_path):
     yamls = []
-    for yaml_name in os.listdir(dir_path):
+    for yaml_name in sorted(os.listdir(dir_path)):
         if yaml_name.startswith("test_") and yaml_name.endswith(".yml"):
             yamls.append(
                 {
@@ -50,7 +93,7 @@ def list_test_yaml(dir_path):
 def get_test_groups():
     test_dirs = os.listdir(SCRIPT_DIR)
     groups = {}
-    for test_group_dir in test_dirs:
+    for test_group_dir in sorted(test_dirs):
         group_dir_path = os.path.join(SCRIPT_DIR, test_group_dir)
         if not os.path.isdir(group_dir_path):
             continue
@@ -65,6 +108,7 @@ def prepare_test(test_name, test_path):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
             kwargs["test_path"] = test_path
+            kwargs["test_name"] = test_name
             return func(*args, **kwargs)
 
         return wrapper
@@ -82,13 +126,29 @@ for group_name, group_tests in get_test_groups().items():
         test_path = test_config["path"]
 
         @pytest.mark.skipif(
-            os.getenv("IPA_SERVER_HOST") is None,
+            not get_server_host(),
             reason="Environment variable IPA_SERVER_HOST must be set",
         )
         @prepare_test(test_name, test_path)
-        def method(self, test_path):
+        def method(self, test_path, test_name):
-            result = run_playbook(test_path)
+            result = run_playbook(test_path, test_name)
-            assert result.returncode == 0
+            status_code_msg = "ansible-playbook return code: {}".format(
+                result.returncode
+            )
+            assert_msg = "\n".join(
+                [
+                    "",
+                    "-" * 30 + " Captured stdout " + "-" * 30,
+                    result.stdout.decode("utf8"),
+                    "-" * 30 + " Captured stderr " + "-" * 30,
+                    result.stderr.decode("utf8"),
+                    "-" * 30 + " Playbook Return Code " + "-" * 30,
+                    status_code_msg,
+                ]
+            )
+            # Need to get the last bytes of msg otherwise Azure
+            #   will cut it out.
+            assert result.returncode == 0, assert_msg[-2500:]
 
         _tests[test_name] = method
-    globals()[group_name] = type(group_name, (TestCase,), _tests)
+    globals()[group_name] = type(group_name, tuple([TestCase]), _tests,)