Merge pull request #380 from seocam/pytest-tests

Added ability to add pytest tests

.gitignore

@@ -4,3 +4,5 @@
 # ignore virtual environments
 /.tox/
 /.venv/
+
+tests/logs/

pytest.ini

@@ -1,3 +1,6 @@
 [pytest]
 python_files = test_*.py
 junit_family = xunit1
+markers=
+    source_order: mark test as order bound
+    playbook: playbook tests

requirements-dev.txt

@@ -0,0 +1,2 @@
+-r requirements-tests.txt
+ipdb

requirements-tests.txt

@@ -0,0 +1,6 @@
+-r requirements.txt
+pytest>=2.7
+pytest-sourceorder>=0.5
+pytest-split-tests>=1.0.3
+testinfra>=5.0
+jmespath>=0.9  # needed for the `json_query` filter

tests/README.md

@@ -2,7 +2,7 @@
 
 ## Before starting
 
-In order to run ansible-freeipa tests you will need to have `ansible`, `pytest` and `jmespath` installed on your machine. We'll call this local machine `controller`. `jmespath` is needed for the `json_query` filter.
+In order to run ansible-freeipa tests you will need to install the dependencies listed in the file `requirements-tests.txt` in your local machine. We'll call this local machine `controller`.
 
 You will also need to have a remote host with freeipa server installed and configured. We'll call this remote host `ipaserver`.
 
@@ -63,6 +63,20 @@ IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest -rs
 
 For a complete list of options check `pytest --help`.
 
+### Types of tests
+
+#### Playbook tests
+
+The playbook tests will run our roles / modules using Ansible with various parameters. Most of these tests will be executed more than once, to verify idempotence. In  general those tests don't verify the state of the machine after the playbook is executed.
+
+To select only these tests use the option `-m "playbook"`
+
+#### Python tests (pytests)
+
+The pytests are tests that will execute small playbooks and then will verify the test results immediately after, using python code for that.
+
+To select only these tests on a test execution use the option `-m "not playbook"`.
+
 
 ## Running tests in a docker container
 

tests/azure/templates/group_tests.yml

@@ -27,3 +27,8 @@ jobs:
     number_of_groups: 3
     build_number: ${{ parameters.build_number }}
     scenario: ${{ parameters.scenario }}
+
+- template: pytest_tests.yml
+  parameters:
+    build_number: ${{ parameters.build_number }}
+    scenario: ${{ parameters.scenario }}

tests/azure/templates/playbook_tests.yml

@@ -21,7 +21,8 @@ parameters:
 
 jobs:
 - job: Test_Group${{ parameters.group_number }}
-  displayName: Run tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
+  displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
+  timeoutInMinutes: 120
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -30,10 +31,10 @@ jobs:
   - script: |
       pip install \
         "molecule[docker]>=3" \
-        "ansible${{ parameters.ansible_version }}" \
-        jmespath \
-        pytest \
-        pytest-split-tests
+        "ansible${{ parameters.ansible_version }}"
+    displayName: Install molecule and Ansible
+
+  - script: pip install -r requirements-tests.txt
     displayName: Install dependencies
 
   - script: |
@@ -46,6 +47,7 @@ jobs:
 
   - script: |
       pytest \
+        -m "playbook" \
         --verbose \
         --color=yes \
         --test-group-count=${{ parameters.number_of_groups }} \

tests/azure/templates/pytest_tests.yml

@@ -0,0 +1,55 @@
+
+parameters:
+- name: build_number
+  type: string
+- name: scenario
+  type: string
+- name: ansible_version
+  type: string
+  default: ">=2.9,<2.10"
+- name: python_version
+  type: string
+  default: 3.6
+
+jobs:
+- job: Test_PyTests
+  displayName: Run pytests on ${{ parameters.scenario }}
+  timeoutInMinutes: 120
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '${{ parameters.python_version }}'
+
+  - script: |
+      pip install \
+        "molecule[docker]>=3" \
+        "ansible${{ parameters.ansible_version }}"
+    displayName: Install molecule and Ansible
+
+  - script: pip install -r requirements-tests.txt
+    displayName: Install dependencies
+
+  - script: |
+      mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils
+      cp -a roles/* ~/.ansible/roles
+      cp -a plugins/modules/* ~/.ansible/library
+      cp -a plugins/module_utils/* ~/.ansible/module_utils
+      molecule create -s ${{ parameters.scenario }}
+    displayName: Setup test container
+
+  - script: |
+      pytest \
+        -m "not playbook" \
+        --verbose \
+        --color=yes \
+        --junit-xml=TEST-results-pytests.xml
+    displayName: Run tests
+    env:
+      IPA_SERVER_HOST: ${{ parameters.scenario }}
+      RUN_TESTS_IN_DOCKER: true
+
+  - task: PublishTestResults@2
+    inputs:
+      mergeTestResults: true
+      testRunTitle: PlaybookTests-Build${{ parameters.build_number }}
+    condition: succeededOrFailed()

tests/pytests/dnszone/playbooks/dnszone_add_multiple_ipv4_ipv6_forwarders.yaml

@@ -0,0 +1,15 @@
+---
+- name: Playbook to ensure the DNS zones is present with multiple forwarder ipv4, ipv6, and port.
+  hosts: ipaserver
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: 04testzone.test
+      forwarders:
+        - ip_address: 192.11.22.33
+        - ip_address: 192.11.22.34
+          port: 23
+        - ip_address: 2001:db8:cafe:1::1
+        - ip_address: 2001:db8:cafe:1::4
+          port: 34

tests/pytests/dnszone/playbooks/dnszone_add_without_forwarder.yaml

@@ -0,0 +1,9 @@
+---
+- name: Playbook to ensure the DNS zones is present without forwarder as well.
+  hosts: ipaserver
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: 01testzone.test
+      forwarders: []

tests/pytests/dnszone/playbooks/dnszone_del_multiple.yaml

@@ -0,0 +1,10 @@
+---
+- name: Playbook to ensure remove multiple dnszone.
+  hosts: ipaserver
+  become: true
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: delzone1.com,delzone2.com,delzone3.com
+      state: absent

tests/pytests/dnszone/playbooks/dnszone_disable.yaml

@@ -0,0 +1,9 @@
+---
+- name: Playbook to ensure the DNS zones disabled.
+  hosts: ipaserver
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: 26testzone.test
+      state: disabled

tests/pytests/dnszone/playbooks/dnszone_enable.yaml

@@ -0,0 +1,9 @@
+---
+- name: Playbook to ensure the DNS zones enabled.
+  hosts: ipaserver
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: 26testzone.test
+      state: enabled

tests/pytests/dnszone/playbooks/dnszone_invalid_ip.yaml

@@ -0,0 +1,11 @@
+---
+- name: Playbook to with invalid IP’s in allow_transfer.
+  hosts: ipaserver
+  become: true
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: invalidzone.test
+      forwarders:
+        - ip_address: in.va.li.d

tests/pytests/dnszone/playbooks/dnszone_invalid_serial.yaml

@@ -0,0 +1,10 @@
+---
+- name: Playbook to ensure, not able to add invalid(more than 4294967295) serial numbers.
+  hosts: ipaserver
+  become: true
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: invalidserialzone.test
+      serial: 429496729599

tests/pytests/dnszone/playbooks/dnszone_name_from_ip.yaml

@@ -0,0 +1,8 @@
+---
+- name: Playbook to ensure reverse zone is added by the IP..
+  hosts: ipaserver
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name_from_ip: 192.8.2.0/22

tests/pytests/dnszone/playbooks/dnszone_with_forward_policy_only.yaml

@@ -0,0 +1,9 @@
+---
+- name: Playbook to ensure the DNS zones is present with forward_policy only.
+  hosts: ipaserver
+
+  tasks:
+  - ipadnszone:
+      ipaadmin_password: SomeADMINpassword
+      name: 26testzone.test
+      forward_policy: only

tests/pytests/dnszone/test_dnszone.py

@@ -0,0 +1,147 @@
+# Authors:
+#   Sergio Oliveira Campos <seocam@redhat.com>
+#
+# Copyright (C) 2020 Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from pytest_sourceorder import ordered
+
+from utils import AnsibleFreeIPATestCase, kinit_admin, kdestroy
+
+BASE_PATH = "pytests/dnszone/playbooks/"
+
+
+@ordered
+class TestDNSZone(AnsibleFreeIPATestCase):
+    def test_dnszone_add_without_forwarder(self):
+        """TC-01: Add dns zone without forwarder."""
+        zone01 = "01testzone.test"
+        self.check_notexists([zone01], "dnszone-find")
+        self.run_playbook(BASE_PATH + "dnszone_add_without_forwarder.yaml")
+        self.check_details([zone01], "dnszone-find")
+
+    def test_dnszone_add_multiple_ipv4_ipv6_forwarders(self):
+        """TC-04: Update multiple ipv4 and ipv6 forwarders."""
+        zone04 = "04testzone.test"
+        self.check_notexists([zone04], "dnszone-find")
+
+        # add dns zone with multiple forwarders
+        self.run_playbook(
+            (BASE_PATH + "dnszone_add_multiple_ipv4_ipv6_forwarders.yaml")
+        )
+
+        exp_forwarders = [
+            "192.11.22.33",
+            "192.11.22.34 port 23",
+            "2001:db8:cafe:1::1",
+            "2001:db8:cafe:1::4 port 34",
+        ]
+        exp_forwarders = ", ".join(exp_forwarders)
+        self.check_details([exp_forwarders], "dnszone-find", [zone04])
+
+    def test_dnszone_with_forward_policy_only(self):
+        """TC-26: Add DNS zone with forward_policy only."""
+        zone26 = "26testzone.test"
+        self.check_notexists([zone26], "dnszone-find")
+        # add dns zone
+        self.run_playbook(BASE_PATH + "dnszone_with_forward_policy_only.yaml")
+        self.check_details(["Forward policy: only"], "dnszone-find", [zone26])
+
+    def test_dnszone_disable(self):
+        """TC-30: Disable DNS Zone."""
+        zone26 = "26testzone.test"
+        self.check_details(["Active zone: TRUE"], "dnszone-find", [zone26])
+        # Disable dns zone
+        self.run_playbook(BASE_PATH + "dnszone_disable.yaml")
+        self.check_details(["Active zone: FALSE"], "dnszone-find", [zone26])
+
+    def test_dnszone_enable(self):
+        """TC-31: Enable DNS Zone."""
+        zone26 = "26testzone.test"
+        self.check_details(["Active zone: FALSE"], "dnszone-find", [zone26])
+        # Enable dns zone
+        self.run_playbook(BASE_PATH + "dnszone_enable.yaml")
+        self.check_details(["Active zone: TRUE"], "dnszone-find", [zone26])
+
+    def test_dnszone_name_from_ip(self):
+        """TC-35: Add dns zone with reverse zone IP. Bug#1845056"""
+        zone = "8.192.in-addr.arpa."
+        expected_msg = "Zone name: {0}".format(zone)
+        self.check_notexists([expected_msg], "dnszone-find", [zone])
+
+        self.mark_xfail_using_ansible_freeipa_version(
+            version="ansible-freeipa-0.1.12-5.el8.noarch",
+            reason="Fix is not available for BZ-1845056",
+        )
+
+        self.run_playbook(BASE_PATH + "dnszone_name_from_ip.yaml")
+        self.check_details([expected_msg], "dnszone-find", [zone])
+
+    def test_dnszone_del_multiple(self):
+        """TC-33: Delete multiple DNS zones Bug#1845058"""
+        zone = ["delzone1.com", "delzone2.com", "delzone3.com"]
+        for add_zone in zone:
+            kinit_admin(self.master)
+            self.master.run("ipa dnszone-add " + add_zone)
+            self.check_details([add_zone], "dnszone-show", [add_zone])
+            kdestroy(self.master)
+
+        self.mark_xfail_using_ansible_freeipa_version(
+            version="ansible-freeipa-0.1.12-5.el8.noarch",
+            reason="Fix is not available for BZ-1845058",
+        )
+
+        self.run_playbook(BASE_PATH + "dnszone_del_multiple.yaml")
+        # verify multiple dnszones are removed
+        for add_zone in zone:
+            error = "ipa: ERROR: {0}.: DNS zone not found".format(add_zone)
+            self.check_notexists([error], "dnszone-show", [add_zone])
+
+    def test_dnszone_invalid_ip(self):
+        """TC-07: Update with invalid IP’s in allow_transfer. Bug#1845051"""
+        invalid_zone_name = "invalidzone.test"
+        invalid_zone_ip = "in.va.li.d"
+        expected_error = "Invalid IP for DNS forwarder"
+
+        self.mark_xfail_using_ansible_freeipa_version(
+            version="ansible-freeipa-0.1.12-5.el8.noarch",
+            reason="Fix is not available for BZ-1845058",
+        )
+
+        self.run_playbook_with_exp_msg(
+            BASE_PATH + "dnszone_invalid_ip.yaml", expected_error,
+        )
+        self.check_notexists(
+            [invalid_zone_ip], "dnszone-show", [invalid_zone_name],
+        )
+
+    def test_invalid_serial(self):
+        """TC-13: Update invalid Serial."""
+        invalid_zone_name = "invalidserialzone.test"
+        invalid_serial = "429496729599"
+        expected_error = "invalid 'serial': can be at most 4294967295"
+
+        self.mark_xfail_using_ansible_freeipa_version(
+            version="ansible-freeipa-0.1.12-5.el8.noarch",
+            reason="Fix is not available for BZ-1845058",
+        )
+
+        self.run_playbook_with_exp_msg(
+            BASE_PATH + "dnszone_invalid_serial.yaml", expected_error
+        )
+        cmd = "dnszone-show"
+        self.check_notexists([invalid_serial], cmd, [invalid_zone_name])

tests/test_playbook_runs.py

@@ -1,117 +1,38 @@
 #!/usr/bin/env python
 
-import os
-import functools
-import tempfile
+# Authors:
+#   Sergio Oliveira Campos <seocam@redhat.com>
+#
+# Copyright (C) 2020 Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import subprocess
+import pytest
+import functools
 
 from unittest import TestCase
 
-import pytest
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-def is_docker_env():
-    if os.getenv("RUN_TESTS_IN_DOCKER", "0") == "0":
-        return False
-    return True
-
-
-def get_ssh_password():
-    return os.getenv("IPA_SSH_PASSWORD")
-
-
-def get_server_host():
-    return os.getenv("IPA_SERVER_HOST")
-
-
-def get_molecule_scenario():
-    return get_server_host()
-
-
-def get_inventory_content():
-    ipa_server_host = get_server_host()
-
-    if is_docker_env():
-        ipa_server_host += " ansible_connection=docker"
-
-    sshpass = get_ssh_password()
-    if sshpass:
-        ipa_server_host += " ansible_ssh_pass=%s" % sshpass
-
-    lines = [
-        "[ipaserver]",
-        ipa_server_host,
-        "[ipaserver:vars]",
-        "ipaserver_domain=test.local",
-        "ipaserver_realm=TEST.LOCAL",
-    ]
-    return "\n".join(lines).encode("utf8")
-
-
-def write_logs(result, test_name):
-    log_dir = os.path.join(SCRIPT_DIR, "logs")
-    if not os.path.exists(log_dir):
-        os.makedirs(log_dir)
-
-    # Write stdout log for test
-    log_path = os.path.join(log_dir, test_name + ".log")
-    with open(log_path, "w") as log_file:
-        log_file.write(result.stdout.decode("utf-8"))
-
-    # Write stderr log for test
-    error_log_path = os.path.join(log_dir, test_name + "-error.log")
-    with open(error_log_path, "w") as log_file:
-        log_file.write(result.stderr.decode("utf-8"))
-
-
-def run_playbook(playbook, test_name):
-    with tempfile.NamedTemporaryFile() as inventory_file:
-        inventory_file.write(get_inventory_content())
-        inventory_file.flush()
-        cmd = [
-            "ansible-playbook",
-            "-i",
-            inventory_file.name,
-            playbook,
-        ]
-        process = subprocess.run(
-            cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        )
-    write_logs(process, test_name)
-
-    return process
-
-
-def list_test_yaml(dir_path):
-    yamls = []
-    for yaml_name in sorted(os.listdir(dir_path)):
-        if yaml_name.startswith("test_") and yaml_name.endswith(".yml"):
-            yamls.append(
-                {
-                    "path": os.path.join(dir_path, yaml_name),
-                    "name": yaml_name.split(".")[0],
-                }
-            )
-    return yamls
-
-
-def get_test_groups():
-    test_dirs = os.listdir(SCRIPT_DIR)
-    groups = {}
-    for test_group_dir in sorted(test_dirs):
-        group_dir_path = os.path.join(SCRIPT_DIR, test_group_dir)
-        if not os.path.isdir(group_dir_path):
-            continue
-        yamls = list_test_yaml(group_dir_path)
-        if yamls:
-            groups[test_group_dir] = yamls
-    return groups
+from utils import get_test_playbooks, get_server_host, run_playbook
 
 
 def prepare_test(test_name, test_path):
+    """Decorator for the tests generated automatically from playbooks.
+
+    Injects 2 arguments to the test (`test_path` and `test_name`) and
+    name the test method using test name (to ensure test reports are useful).
+    """
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
@@ -127,36 +48,20 @@ def prepare_test(test_name, test_path):
 
 # Dynamically create the TestCase classes with respective
 #   test_* methods.
-for group_name, group_tests in get_test_groups().items():
+for test_dir_name, playbooks_in_dir in get_test_playbooks().items():
     _tests = {}
-    for test_config in group_tests:
-        test_name = test_config["name"].replace("-", "_")
-        test_path = test_config["path"]
+    for playbook in playbooks_in_dir:
+        test_name = playbook["name"].replace("-", "_")
+        test_path = playbook["path"]
 
         @pytest.mark.skipif(
             not get_server_host(),
             reason="Environment variable IPA_SERVER_HOST must be set",
         )
+        @pytest.mark.playbook
         @prepare_test(test_name, test_path)
         def method(self, test_path, test_name):
-            result = run_playbook(test_path, test_name)
-            status_code_msg = "ansible-playbook return code: {}".format(
-                result.returncode
-            )
-            assert_msg = "\n".join(
-                [
-                    "",
-                    "-" * 30 + " Captured stdout " + "-" * 30,
-                    result.stdout.decode("utf8"),
-                    "-" * 30 + " Captured stderr " + "-" * 30,
-                    result.stderr.decode("utf8"),
-                    "-" * 30 + " Playbook Return Code " + "-" * 30,
-                    status_code_msg,
-                ]
-            )
-            # Need to get the last bytes of msg otherwise Azure
-            #   will cut it out.
-            assert result.returncode == 0, assert_msg[-2500:]
+            run_playbook(test_path)
 
         _tests[test_name] = method
-    globals()[group_name] = type(group_name, tuple([TestCase]), _tests,)
+    globals()[test_dir_name] = type(test_dir_name, tuple([TestCase]), _tests,)

tests/utils.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+
+# Authors:
+#   Sergio Oliveira Campos <seocam@redhat.com>
+#
+# Copyright (C) 2020 Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import pytest
+import subprocess
+import tempfile
+import testinfra
+
+from unittest import TestCase
+
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def is_docker_env():
+    if os.getenv("RUN_TESTS_IN_DOCKER", "0") == "0":
+        return False
+    return True
+
+
+def get_ssh_password():
+    return os.getenv("IPA_SSH_PASSWORD")
+
+
+def get_server_host():
+    return os.getenv("IPA_SERVER_HOST")
+
+
+def get_inventory_content():
+    """Create the content of an inventory file for a test run."""
+    ipa_server_host = get_server_host()
+
+    if is_docker_env():
+        ipa_server_host += " ansible_connection=docker"
+
+    sshpass = get_ssh_password()
+    if sshpass:
+        ipa_server_host += " ansible_ssh_pass=%s" % sshpass
+
+    lines = [
+        "[ipaserver]",
+        ipa_server_host,
+        "[ipaserver:vars]",
+        "ipaserver_domain=test.local",
+        "ipaserver_realm=TEST.LOCAL",
+    ]
+    return "\n".join(lines).encode("utf8")
+
+
+def get_test_name_from_playbook_path(playbook):
+    """
+    Create a test name based of a playbook path.
+
+    For example:
+        Input: /home/johndoe/ansible-freeipa/tests/dnszone/test_dnszone_mod.yml
+        Output: dnszone_test_dnszone_mod
+    """
+    playbook_abspath = os.path.abspath(playbook)
+    playbook_rel_to_tests_dir = playbook_abspath.replace(SCRIPT_DIR, "")
+    playbook_slug = playbook_rel_to_tests_dir.strip("/").replace("/", "_")
+    return os.path.splitext(playbook_slug)[0]
+
+
+def write_logs(result, test_name):
+    """Write logs of a ansible run logs to `test/logs/`."""
+    log_dir = os.path.join(SCRIPT_DIR, "logs")
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+
+    # Write stdout log for test
+    log_path = os.path.join(log_dir, "ansible_" + test_name + ".log")
+    with open(log_path, "w") as log_file:
+        log_file.write(result.stdout.decode("utf-8"))
+
+    # Write stderr log for test
+    error_log_path = os.path.join(log_dir, test_name + "-error.log")
+    with open(error_log_path, "w") as log_file:
+        log_file.write(result.stderr.decode("utf-8"))
+
+
+def _run_playbook(playbook):
+    """
+    Create a inventory using a temporary file and run ansible using it.
+
+    The logs of the run will be placed in `tests/logs/`.
+    """
+    with tempfile.NamedTemporaryFile() as inventory_file:
+        inventory_file.write(get_inventory_content())
+        inventory_file.flush()
+        cmd = [
+            "ansible-playbook",
+            "-i",
+            inventory_file.name,
+            playbook,
+        ]
+        process = subprocess.run(
+            cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+    test_name = get_test_name_from_playbook_path(playbook)
+    write_logs(process, test_name)
+
+    return process
+
+
+def run_playbook(playbook, allow_failures=False):
+    """
+    Run an Ansible playbook and assert the return code.
+
+    Call ansible (using _run_playbook function) and assert the result of
+    the execution.
+
+    In case of failure the tail of the error message will be displayed
+    as an assertion message.
+
+    The full log of the execution will be available in the directory
+    `tests/logs/`.
+    """
+    result = _run_playbook(playbook)
+
+    if allow_failures:
+        return result
+
+    status_code_msg = "ansible-playbook return code: {}".format(
+        result.returncode
+    )
+    assert_msg = "\n".join(
+        [
+            "",
+            "-" * 30 + " Captured stdout " + "-" * 30,
+            result.stdout.decode("utf8"),
+            "-" * 30 + " Captured stderr " + "-" * 30,
+            result.stderr.decode("utf8"),
+            "-" * 30 + " Playbook Return Code " + "-" * 30,
+            status_code_msg,
+        ]
+    )
+
+    # Need to get the last bytes of msg otherwise Azure
+    #   will cut it out.
+    assert result.returncode == 0, assert_msg[-2500:]
+
+    return result
+
+
+def list_test_yaml(dir_path):
+    """
+    List the test playbooks inside a given directory.
+
+    A test playbook is any file inside the directory which the name starts with
+    `test_` and the extension is `.yml`.
+    """
+    yamls = []
+    for root, dirs, files in os.walk(dir_path):
+        for yaml_name in files:
+            if yaml_name.startswith("test_") and yaml_name.endswith(".yml"):
+                test_yaml_path = os.path.join(root, yaml_name)
+                yamls.append(
+                    {
+                        "path": test_yaml_path,
+                        "name": yaml_name.split(".")[0],
+                    }
+                )
+    return yamls
+
+
+def get_test_playbooks():
+    """
+    Get playbook tests grouped by first level directory.
+
+    This function visits the first level of directories inside `tests/` and
+    look for ansible playbooks on them.
+
+    Returns a dict with the directories found in `tests/` as key and a
+    list of test playbook files inside of it.
+
+    A test playbook is any file inside the directory which the name starts with
+    `test_` and the extension is `.yml`.
+    """
+    test_dirs = os.listdir(SCRIPT_DIR)
+    groups = {}
+    for test_group_dir in sorted(test_dirs):
+        group_dir_path = os.path.join(SCRIPT_DIR, test_group_dir)
+        if not os.path.isdir(group_dir_path):
+            continue
+        yamls = list_test_yaml(group_dir_path)
+        if yamls:
+            groups[test_group_dir] = yamls
+    return groups
+
+
+def kinit_admin(host, admin="admin", password="SomeADMINpassword"):
+    return host.run_test("kinit " + admin + "<<< " + password)
+
+
+def kdestroy(host):
+    return host.run_test("kdestroy -A")
+
+
+class AnsibleFreeIPATestCase(TestCase):
+    def setUp(self):
+        if is_docker_env():
+            protocol = "docker://"
+            user = ""
+        else:
+            protocol = "ssh://"
+
+            password = get_ssh_password() or ""
+            if password:
+                password = ":" + password
+
+            current_user = os.getenv("USER")
+            ansible_user = os.getenv("ANSIBLE_REMOTE_USER", current_user)
+            user = ansible_user + password + "@"
+
+        host_connection_info = protocol + user + get_server_host()
+        self.master = testinfra.get_host(host_connection_info)
+
+    def run_playbook(self, playbook, allow_failures=False):
+        return run_playbook(playbook, allow_failures)
+
+    def run_playbook_with_exp_msg(self, playbook, expected_msg):
+        result = self.run_playbook(playbook, allow_failures=True)
+        assert (
+            expected_msg in result.stdout.decode("utf8")
+            or
+            expected_msg in result.stderr.decode("utf8")
+        )
+
+    def check_details(self, expected_output, cmd, extra_cmds=None):
+        cmd = "ipa " + cmd
+        if extra_cmds:
+            cmd += " " + " ".join(extra_cmds)
+        kinit_admin(self.master)
+        res = self.master.run(cmd)
+        if res.rc != 0:
+            for output in expected_output:
+                assert output in res.stderr
+        else:
+            for output in expected_output:
+                assert output in res.stdout
+        kdestroy(self.master)
+
+    def check_notexists(self, members, cmd, extra_cmds=None):
+        cmd = "ipa " + cmd
+        if extra_cmds:
+            cmd += " " + " ".join(extra_cmds)
+        kinit_admin(self.master)
+        res = self.master.run(cmd)
+        for member in members:
+            assert member not in res.stdout
+        kdestroy(self.master)
+
+    def mark_xfail_using_ansible_freeipa_version(self, version, reason):
+        package = self.master.package("ansible-freeipa")
+
+        if not package.is_installed:
+            return
+
+        if package.version == version:
+            pytest.xfail(reason)