Merge pull request #1276 from t-woerner/run_containers_ubuntu_24_04_test

Run tests with podman and ubuntu 20.04

.gitignore

@@ -12,3 +12,4 @@ importer_result.json
 /.venv/
 
 tests/logs/
+TEST*.xml

tests/README.md

@@ -36,6 +36,12 @@ environment variable. For example:
 IPA_SSH_PASSWORD=<ipaserver_ssh_password> IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
 ```
 
+If you want, or need to to set the Python interpreter to use, you must set `IPA_PYTHON_PATH`
+environment variable. For example:
+
+```
+IPA_PYTHON_PATH=/usr/bin/python3.14 IPA_SERVER_HOST=<ipaserver_host_or_ip> pytest
+```
 
 To run a single test use the full path with the following format:
 

tests/azure/pr-pipeline.yml

@@ -15,8 +15,8 @@ stages:
   - template: templates/fast_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: fedora-latest
+      distro: fedora-latest
-      ansible_version: "-core >=2.14,<2.15"
+      ansible_version: "-core >=2.15,<2.16"
 
 # Galaxy on Fedora
 
@@ -26,8 +26,8 @@ stages:
   - template: templates/fast_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: fedora-latest
+      distro: fedora-latest
-      ansible_version: "-core >=2.14,<2.15"
+      ansible_version: "-core >=2.15,<2.16"
 
 # CentOS 9 Stream
 
@@ -37,8 +37,8 @@ stages:
   - template: templates/fast_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: c9s
+      distro: c9s
-      ansible_version: "-core >=2.14,<2.15"
+      ansible_version: "-core >=2.15,<2.16"
 
 # CentOS 8 Stream
 
@@ -48,19 +48,23 @@ stages:
   - template: templates/fast_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: c8s
+      distro: c8s
-      ansible_version: "-core >=2.14,<2.15"
+      ansible_version: "-core >=2.15,<2.16"
+      target_python: "/usr/libexec/platform-python"
 
+# CentOS 7 cannot be used with current systemd
+#
 # CentOS 7
-
+#
-- stage: CentOS_7
+# - stage: CentOS_7
-  dependsOn: []
+#   dependsOn: []
-  jobs:
+#   jobs:
-  - template: templates/fast_tests.yml
+#   - template: templates/fast_tests.yml
-    parameters:
+#     parameters:
-      build_number: $(Build.BuildNumber)
+#       build_number: $(Build.BuildNumber)
-      scenario: centos-7
+#       distro: centos-7
-      ansible_version: "-core >=2.14,<2.15"
+#       ansible_version: "-core >=2.15,<2.16"
+#       target_python: "/usr/bin/python2"
 
 # Rawhide
 
@@ -70,5 +74,5 @@ stages:
   - template: templates/fast_tests.yml
     parameters:
       build_number: $(Build.BuildNumber)
-      scenario: fedora-rawhide
+      distro: fedora-rawhide
-      ansible_version: "-core >=2.14,<2.15"
+      ansible_version: "-core >=2.15,<2.16"

tests/azure/templates/fast_tests.yml

@@ -1,6 +1,6 @@
 ---
 parameters:
-  - name: scenario
+  - name: distro
     type: string
     default: fedora-latest
   - name: build_number
@@ -8,6 +8,9 @@ parameters:
   - name: ansible_version
     type: string
     default: ""
+  - name: target_python
+    type: string
+    default: "/usr/bin/python3"
 
 jobs:
 - template: playbook_fast.yml
@@ -15,13 +18,14 @@ jobs:
     group_number: 1
     number_of_groups: 1
     build_number: ${{ parameters.build_number }}
-    scenario: ${{ parameters.scenario }}
+    distro: ${{ parameters.distro }}
     ansible_version: ${{ parameters.ansible_version }}
     python_version: '< 3.12'
+    target_python: ${{ parameters.target_python }}
 
 # - template: pytest_tests.yml
 #   parameters:
 #     build_number: ${{ parameters.build_number }}
-#     scenario: ${{ parameters.scenario }}
+#     distro: ${{ parameters.distro }}
 #     ansible_version: ${{ parameters.ansible_version }}
 #     python_version: '< 3.12'

tests/azure/templates/playbook_fast.yml

@@ -6,7 +6,7 @@ parameters:
   - name: number_of_groups
     type: number
     default: 1
-  - name: scenario
+  - name: distro
     type: string
     default: fedora-latest
   - name: ansible_version
@@ -17,28 +17,28 @@ parameters:
     default: 3.x
   - name: build_number
     type: string
+  - name: target_python
+    type: string
+    default: "/usr/bin/python3"
 
 jobs:
 - job: Test_Group${{ parameters.group_number }}
-  displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
+  displayName: Run playbook tests ${{ parameters.distro }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }})
   timeoutInMinutes: 360
   variables:
   - template: variables.yaml
-  - template: variables_${{ parameters.scenario }}.yaml
+  - template: variables_${{ parameters.distro }}.yaml
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '${{ parameters.python_version }}'
 
   - script: |
-      pip install \
+      pip install "ansible${{ parameters.ansible_version }}"
-        "molecule-plugins[docker]" \
-        "requests<2.29" \
-        "ansible${{ parameters.ansible_version }}"
     retryCountOnTaskFailure: 5
-    displayName: Install molecule and Ansible
+    displayName: Install Ansible
 
-  - script: ansible-galaxy collection install community.docker ansible.posix
+  - script: ansible-galaxy collection install containers.podman
     retryCountOnTaskFailure: 5
     displayName: Install Ansible collections
 
@@ -47,43 +47,35 @@ jobs:
     displayName: Install dependencies
 
   - script: |
-      rm -rf ~/ansible
+      . utils/set_test_modules
-      mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils
+      python3 utils/check_test_configuration.py ${{ parameters.distro }}
-      cp -a roles/* ~/.ansible/roles
+    displayName: Check test configuration
-      cp -a plugins/modules/* ~/.ansible/library
+
-      cp -a plugins/module_utils/* ~/.ansible/module_utils
+  - script: |
-      molecule create -s ${{ parameters.scenario }}
+      utils/setup_test_container.sh \
-    retryCountOnTaskFailure: 5
+        -e podman \
-    displayName: Setup test container
+        -a \
-    env:
+        -m 4 \
-      ANSIBLE_LIBRARY: ./molecule
+        -n "ipaserver.test.local" \
+        -p ${{ parameters.target_python }} \
+        -i ${{ parameters.distro }}-server \
+        ${{ parameters.distro }}-test
+    displayName: Setup target container
 
   - script: |
       . utils/set_test_modules
-      python utils/check_test_configuration.py ${{ parameters.scenario }}
+      pytest -m "playbook" --verbose --color=yes --suppress-no-test-exit-code --junit-xml=TEST-results-pr-check.xml
-    displayName: Check scenario test configuration
-
-  - script: |
-      . utils/set_test_modules
-      if ! pytest \
-        -m "playbook" \
-        --verbose \
-        --color=yes \
-        --suppress-no-test-exit-code \
-        --splits=${{ parameters.number_of_groups }} \
-        --group=${{ parameters.group_number }} \
-        --randomly-seed=$(date "+%Y%m%d") \
-        --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml
-      then
-        [ $? -eq 5 ] && true || false
-      fi
     displayName: Run playbook tests
     env:
-      IPA_SERVER_HOST: ${{ parameters.scenario }}
+      ANSIBLE_ROLES_PATH: "${PWD}/roles"
-      RUN_TESTS_IN_DOCKER: true
+      ANSIBLE_LIBRARY: "${PWD}/plugins"
+      ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils"
+      IPA_SERVER_HOST: ${{ parameters.distro }}-test
+      RUN_TESTS_IN_DOCKER: podman
       IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }}
       IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }}
       IPA_VERBOSITY: "-vvv"
+      IPA_PYTHON_PATH: ${{ parameters.target_python }}
 
   - task: PublishTestResults@2
     inputs:

tests/utils.py

@@ -43,6 +43,10 @@ def get_ssh_password():
     return os.getenv("IPA_SSH_PASSWORD")
 
 
+def get_python_interpreter():
+    return os.getenv("IPA_PYTHON_PATH")
+
+
 def get_server_host():
     return os.getenv("IPA_SERVER_HOST")
 
@@ -97,6 +101,12 @@ def get_inventory_content():
     if sshpass:
         ipa_server_host += " ansible_ssh_pass=%s" % sshpass
 
+    python_interpreter = get_python_interpreter()
+    if python_interpreter:
+        ipa_server_host += (
+            " ansible_python_interpreter=%s" % python_interpreter
+        )
+
     lines = [
         "[ipaserver]",
         ipa_server_host,
@@ -138,30 +148,6 @@ def write_logs(result, test_name):
         log_file.write(result.stderr.decode("utf-8"))
 
 
-def _run_playbook(playbook):
-    """
-    Create a inventory using a temporary file and run ansible using it.
-
-    The logs of the run will be placed in `tests/logs/`.
-    """
-    with tempfile.NamedTemporaryFile() as inventory_file:
-        inventory_file.write(get_inventory_content())
-        inventory_file.flush()
-        cmd_options = ["-i", inventory_file.name]
-        verbose = os.environ.get("IPA_VERBOSITY", None)
-        if verbose is not None:
-            cmd_options.append(verbose)
-        cmd = ["ansible-playbook"] + cmd_options + [playbook]
-        # pylint: disable=subprocess-run-check
-        process = subprocess.run(
-            cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        )
-    test_name = get_test_name_from_playbook_path(playbook)
-    write_logs(process, test_name)
-
-    return process
-
-
 def _truncate(lines, charcount, minlines=0):
     output = ""
     line_count = 1
@@ -178,12 +164,11 @@ def _truncate(lines, charcount, minlines=0):
     return output
 
 
-def run_playbook(playbook, allow_failures=False):
+def _run_playbook(playbook):
     """
-    Run an Ansible playbook and assert the return code.
+    Create a inventory using a temporary file and run ansible using it.
 
-    Call ansible (using _run_playbook function) and assert the result of
+    The logs of the run will be placed in `tests/logs/`.
-    the execution.
 
     In case of failure the tail of the error message will be displayed
     as an assertion message.
@@ -191,43 +176,65 @@ def run_playbook(playbook, allow_failures=False):
     The full log of the execution will be available in the directory
     `tests/logs/`.
     """
-    result = _run_playbook(playbook)
+    with tempfile.NamedTemporaryFile() as inventory_file:
+        inventory_file.write(get_inventory_content())
+        inventory_file.flush()
+        cmd_options = ["-i", inventory_file.name]
+        verbose = os.environ.get("IPA_VERBOSITY", None)
+        if verbose is not None:
+            cmd_options.append(verbose)
+        cmd = ["ansible-playbook"] + cmd_options + [playbook]
+        process = subprocess.run(
+            cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE, check=False
+        )
+    test_name = get_test_name_from_playbook_path(playbook)
+    write_logs(process, test_name)
 
-    if allow_failures:
+    msg = ""
-        return result
+    if process.returncode != 0:
+        status_code_msg = "ansible-playbook return code: {0}".format(
+            process.returncode
+        )
 
-    status_code_msg = "ansible-playbook return code: {0}".format(
+        _stdout = process.stdout.decode("utf8")
-        result.returncode
+        _stderr = process.stderr.decode("utf8")
-    )
+        # Truncate stdout and stderr in the way that it hopefully
-    _stdout = result.stdout.decode("utf8")
+        # shows all important information. At least 15 lines of stdout
-    _stderr = result.stderr.decode("utf8")
+        # (Ansible tasks) and remaining from stderr to fill up to
-    # Truncate stdout and stderr in the way that it hopefully
+        # maxlen size.
-    # shows all important information. At least 15 lines of stdout
+        maxlen = 2000
-    # (Ansible tasks) and remaining from stderr to fill up to
+        factor = maxlen / (len(_stdout) + len(_stderr))
-    # maxlen size.
+        stdout = _truncate(_stdout.splitlines(),
-    maxlen = 2000
+                           int(factor * len(_stdout)),
-    factor = maxlen / (len(_stdout) + len(_stderr))
+                           minlines=15)
-    stdout = _truncate(_stdout.splitlines(),
+        stderr = _truncate(_stderr.splitlines(), maxlen - len(stdout))
-                       int(factor * len(_stdout)),
-                       minlines=15)
-    stderr = _truncate(_stderr.splitlines(), maxlen - len(stdout))
 
-    assert_msg = "\n".join(
+        msg = "\n".join(
-        [
+            [
-            "",
+                "",
-            "-" * 30 + " Captured stdout " + "-" * 30,
+                "-" * 30 + " Captured stdout " + "-" * 30,
-            stdout,
+                stdout,
-            "-" * 30 + " Captured stderr " + "-" * 30,
+                "-" * 30 + " Captured stderr " + "-" * 30,
-            stderr,
+                stderr
-            "-" * 30 + " Playbook Return Code " + "-" * 30,
+            ]
-            status_code_msg,
+        )
-        ]
+        msg += "-" * 30 + " Playbook Return Code " + "-" * 30 + "\n"
-    )
+        msg += status_code_msg
 
-    # Need to get the last bytes of msg otherwise Azure
+    return process, msg
-    #   will cut it out.
-    assert result.returncode == 0, assert_msg[-2500:]
 
+
+def run_playbook(playbook, allow_failures=False):
+    """
+    Run an Ansible playbook and assert the return code.
+
+    Call ansible (using _run_playbook function) and assert the result of
+    the execution.
+    """
+    result, assert_msg = _run_playbook(playbook)
+    if not allow_failures:
+        assert result.returncode == 0, assert_msg
     return result
 
 

utils/run-tests.sh

@@ -1,38 +1,19 @@
 #!/bin/bash -eu
 
-trap interrupt_exception SIGINT
+SCRIPTDIR="$(readlink -f "$(dirname "$0")")"
+TOPDIR="$(readlink -f "${SCRIPTDIR}/..")"
 
-RST="\033[0m"
+# shellcheck source=utils/shfun
-RED="\033[31m"
+. "${SCRIPTDIR}/shfun"
-# BRIGHTRED="\033[31;1m"
+# shellcheck source=utils/shcontainer
-# GREEN="\033[32m"
+. "${SCRIPTDIR}/shcontainer"
-BRIGHTGREEN="\033[32;1m"
+# shellcheck source=utils/shansible
-# BROWN="\033[33m"
+. "${SCRIPTDIR}/shansible"
-YELLOW="\033[33;1m"
-# NAVY="\033[34m"
-BLUE="\033[34;1m"
-# MAGENTA="\033[35m"
-# BRIGHTMAGENTA="\033[35;1m"
-# DARKCYAN="\033[36m"
-# CYAN="\033[36;1m"
-# BLACK="\033[30m"
-# DARKGRAY="\033[30;1m"
-# GRAY="\033[37m"
-WHITE="\033[37;1m"
-
-TOPDIR="$(readlink -f "$(dirname "$0")/..")"
-
-interrupt_exception() {
-    trap - SIGINT
-    log warn "User interrupted test execution."
-    cleanup
-    exit 1
-}
 
 usage() {
     local prog="${0##*/}"
     cat <<EOF
-usage: ${prog} [-h] [-l] [-e] [-K] [-c CONTAINER] [-s TESTS_SUITE] [-x] [-A SEED.GRP] [-i IMAGE] [-m MEMORY] [-v...] [TEST...]
+usage: ${prog} [-h] [-l] [-e] [-K] [-A|-a ANSIBLE] [-p INTERPRETER] [-c CONTAINER] [-s TESTS_SUITE] [-x] [-S SEED.GRP] [-i IMAGE] [-m MEMORY] [-v...] [TEST...]
     ${prog} runs playbook(s) TEST using an ansible-freeipa testing image.
 
 EOF
@@ -47,172 +28,72 @@ positional arguments:
 
 optional arguments:
   -h              display this message and exit
+  -a ANSIBLE      Ansible version to use, e.g. "ansible-core==2.16.0"
+                  (default: latest ansible-core for the python version)
+  -A              Do not install Ansible, use host's provided one.
   -c CONTAINER    use container CONTAINER to run tests
   -K              keep container, even if tests succeed
   -l              list available images
   -e              force recreation of the virtual environment
   -i IMAGE        select image to run the tests (default: fedora-latest)
-  -m              container memory, in GiB (default: 3)
+  -m MEMORY       container memory, in GiB (default: 3)
+  -p INTERPRETER  Python interpreter to use on target container
   -s TEST_SUITE   run all playbooks for test suite, which is a directory
                   under ${WHITE}tests${RST}
-  -A SEED.GROUP   Replicate Azure's test group and seed (seed is YYYYMMDD)
+  -S SEED.GROUP   Replicate Azure's test group and seed (seed is YYYYMMDD)
   -v              Increase Ansible verbosity (can be used multiple times)
   -x              Stop on first error.
 EOF
 )"
 }
 
-log() {
-    local level="${1^^}" message="${*:2}"
-    case "${level}" in
-        ERROR) COLOR="${RED}" ;;
-        WARN)  COLOR="${YELLOW}" ;;
-        DEBUG) COLOR="${BLUE}" ;;
-        INFO) COLOR="${WHITE}" ;;
-        SUCCESS) COLOR="${BRIGHTGREEN}" ;;
-        *) COLOR="${RST}" ;;
-    esac
-    echo -en "${COLOR}"
-    [ "${level}" == "ERROR" ] && echo -en "${level}:"
-    echo -e "${message}${RST}"
-}
-
-quiet() {
-     "$@" >/dev/null 2>&1
-}
-
-in_python_virtualenv() {
-    local script
-    read -r -d "" script <<EOS
-import sys;
-base = getattr(sys, "base_prefix", ) or getattr(sys, "real_prefix", ) or sys.prefix
-print('yes' if sys.prefix != base else 'no')
-EOS
-    test "$(python -c "${script}")" == "yes"
-}
-
-run_inline_playbook() {
-    local playbook
-    local err
-    quiet mkdir -p "${test_env}/playbooks"
-    playbook=$(mktemp "${test_env}/playbooks/ansible-freeipa-test-playbook_ipa.XXXXXXXX")
-    cat - >"${playbook}"
-    ansible-playbook -i "${inventory}" "${playbook}"
-    err=$?
-    rm "${playbook}"
-    return ${err}
-}
-
-die() {
-    usg="N"
-    if [ "${1}" == "-u" ]
-    then
-       usg="Y"
-       shift 1
-    fi
-    log error "${*}"
-    STOP_CONTAINER="N"
-    cleanup
-    [ "${usg}" == "Y" ] && usage
-    exit 1
-}
-
-make_inventory() {
-    local scenario=$1 engine=${2:-podman}
-    inventory="${test_env}/inventory"
-    log info "Inventory file: ${inventory}"
-    cat << EOF > "${inventory}"
-[ipaserver]
-${scenario} ansible_connection=${engine}
-[ipaserver:vars]
-ipaserver_domain = test.local
-ipaserver_realm = TEST.LOCAL
-EOF
-}
-
-stop_container() {
-    local scenario=${1} engine=${2:-podman}
-    echo "Stopping container..."
-    quiet "${engine}" stop "${scenario}"
-    echo "Removing container..."
-    quiet "${engine}" rm "${scenario}"
-}
-
-cleanup() {
-    if [ $# -gt 0 ]
-    then
-        if [ "${STOP_CONTAINER}" != "N" ]
-        then
-            stop_container "${1}" "${2}"
-            rm "${inventory}"
-        else
-            log info "Keeping container: $(podman ps --format "{{.Names}} - {{.ID}}" --filter "name=${1}")"
-        fi
-    fi
-    if [ "${STOP_VIRTUALENV}" == "Y" ]
-    then
-        echo "Deactivating virtual environment"
-        deactivate
-    fi
-}
-
-list_images() {
-    local quay_api="https://quay.io/api/v1/repository/ansible-freeipa/upstream-tests/tag"
-    echo -e "${WHITE}Available images:"
-    curl --silent -L "${quay_api}" | jq '.tags[]|.name' | tr -d '"'| sort | uniq | sed "s/.*/    &/"
-    echo -e "${RST}"
-}
 
 # Defaults
-
-ANSIBLE_VERSION=${ANSIBLE_VERSION:-'ansible-core'}
 verbose=""
-FORCE_ENV="N"
+engine="${engine:-"podman"}"
 CONTINUE_ON_ERROR=""
 STOP_CONTAINER="Y"
 STOP_VIRTUALENV="N"
 declare -a ENABLED_MODULES
 declare -a ENABLED_TESTS
-ENABLED_MODULES=()
+read -r -a ENABLED_MODULES <<< "${IPA_ENABLED_MODULES:-""}"
-ENABLED_TESTS=()
+read -r -a ENABLED_TESTS <<< "${IPA_ENABLED_MODULES:-""}"
-test_env="${TESTENV_DIR:-${VIRTUAL_ENV:-/tmp/ansible-freeipa-tests}}"
-engine="podman"
-IMAGE_REPO="quay.io/ansible-freeipa/upstream-tests"
 IMAGE_TAG="fedora-latest"
-scenario=""
+scenario="freeipa-tests"
 MEMORY=3
-hostname="ipaserver.test.local"
+IPA_HOSTNAME="ipaserver.test.local"
-SEED=""
+SEED="$(date "+%Y%m%d")"
-GROUP=0
+GROUP=1
 SPLITS=0
-ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"containers.podman"}
+ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"${engine_collection}"}
-
+SKIP_ANSIBLE=""
+ansible_interpreter="/usr/bin/python3"
 EXTRA_OPTIONS=""
+unset ANSIBLE_VERSION
 
 # Process command options
 
-while getopts ":hA:c:ei:Klms:vx" option
+while getopts ":ha:Ac:ei:Klm:p:s:S:vx" option
 do
     case "$option" in
         h) help && exit 0 ;;
         A)
-            [ ${#ENABLED_MODULES[@]} -eq 0 ] || die -u "Can't use '-A' with '-s'"
+           [ -n "${ANSIBLE_VERSION:-""}" ] && die "Can't use -A with '-a'"
-            SEED="$(cut -d. -f1 <<< "${OPTARG}" | tr -d "-")"
+           SKIP_ANSIBLE="YES"
-            GROUP="$(cut -d. -f2 <<< "${OPTARG}")"
+           ;;
-            if [ -z "${SEED}" ] || [ -z "${GROUP}" ]
+        a) 
-            then
+           [ "${SKIP_ANSIBLE:-"no"}" == "YES" ] && die "Can't use -A with '-a'"
-                die -u "Seed for '-A' must have the format YYYYMMDD.N"
+           ANSIBLE_VERSION="${OPTARG}"
-            fi
+           ;;
-            SPLITS=3
-        ;;
         c) scenario="${OPTARG}" ;;
         e) FORCE_ENV="Y" ;;
         i) IMAGE_TAG="${OPTARG}" ;;
         K) STOP_CONTAINER="N" ;;
-        l) list_images && exit 0 || exit 1;;
+        l) "${SCRIPTDIR}"/setup_test_container.sh -l && exit 0 || exit 1 ;;
         m) MEMORY="${OPTARG}" ;;
+        p) ansible_interpreter="${OPTARG}" ;;
         s)
-           [ ${SPLITS} -ne 0 ] && die -u "Can't use '-A' with '-s'"
+           [ ${SPLITS} -ne 0 ] && die -u "Can't use '-S' with '-s'"
            if [ -d "${TOPDIR}/tests/${OPTARG}" ]
            then
                ENABLED_MODULES+=("${OPTARG}")
@@ -220,6 +101,16 @@ do
                log error "Invalid suite: ${OPTARG}"
            fi
            ;;
+        S)
+           [ ${#ENABLED_MODULES[@]} -eq 0 ] || die -u "Can't use '-A' with '-s'"
+           SEED="$(cut -d. -f1 <<< "${OPTARG}" | tr -d "-")"
+           GROUP="$(cut -d. -f2 <<< "${OPTARG}")"
+           if [ -z "${SEED}" ] || [ -z "${GROUP}" ]
+           then
+               die -u "Seed for '-A' must have the format YYYYMMDD.N"
+           fi
+           SPLITS=3
+           ;;
         v) verbose=${verbose:--}${option} ;;
         x) EXTRA_OPTIONS="$EXTRA_OPTIONS --exitfirst" ;;
         *) die -u "Invalid option: ${OPTARG}" ;;
@@ -240,155 +131,28 @@ done
 
 [ ${SPLITS} -eq 0 ] && [ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined."
 
+export STOP_CONTAINER FORCE_ENV STOP_VIRTUALENV ansible_interpreter
+
+# Ensure $python is set
+[ -z "${python}" ] && python="python3"
+
+log info "Controller Python executable: ${python}"
+${python} --version
+
 # Prepare virtual environment
-VENV=$(in_python_virtualenv && echo Y || echo N)
+start_virtual_environment
+log info "Installing dependencies from 'requirements-tests.txt'"
+pip install --upgrade -r "${TOPDIR}/requirements-tests.txt"
 
-if [ "${FORCE_ENV}" == "Y" ]
+[ -z "${SKIP_ANSIBLE}" ] && install_ansible "${ANSIBLE_VERSION:-"ansible-core"}"
-then
-    [ "${VENV}" == "Y" ] && deactivate
-    VENV="N"
-    rm -rf "$test_env"
-    log info "Virtual environment will be (re)created."
-fi
-
-if [ "$VENV" == "N" ]
-then
-    log info "Preparing virtual environment: ${test_env}"
-    if [ ! -d "${test_env}" ]
-    then
-        log info "Creating virtual environment: ${test_env}..."
-        if ! python3 -m venv "${test_env}"
-        then
-            die "Cannot create virtual environment."
-        fi
-    fi
-    if [ -f "${test_env}/bin/activate" ]
-    then
-        log info "Starting virtual environment: ${test_env}"
-        # shellcheck disable=SC1091
-        . "${test_env}/bin/activate" || die "Cannot activate environment."
-        STOP_VIRTUALENV="Y"
-    else
-        die "Cannot activate environment."
-    fi
-    log info "Installing required tools."
-    log none "Upgrading: pip setuptools wheel"
-    pip install --quiet --upgrade pip setuptools wheel
-    log info "Installing dependencies from 'requirements-tests.txt'"
-    pip install --quiet -r "${TOPDIR}/requirements-tests.txt"
-    log info "Installing Ansible: ${ANSIBLE_VERSION}"
-    pip install --quiet "${ANSIBLE_VERSION}"
-    log debug "Ansible version: $(ansible --version | sed -n "1p")${RST}"
-else
-   log info "Using current virtual environment."
-fi
-
-if [ -n "${ANSIBLE_COLLECTIONS}" ]
-then
-    log warn "Installed collections will not be removed after execution."
-    log none "Installing: Ansible Collection ${ANSIBLE_COLLECTIONS}"
-    # shellcheck disable=SC2086
-    quiet ansible-galaxy collection install ${ANSIBLE_COLLECTIONS} || die "Failed to install Ansible collections."
-fi
 
 # Ansible configuration
 export ANSIBLE_ROLES_PATH="${TOPDIR}/roles"
-export ANSIBLE_LIBRARY="${TOPDIR}/plugins:${TOPDIR}/molecule"
+export ANSIBLE_LIBRARY="${TOPDIR}/plugins"
 export ANSIBLE_MODULE_UTILS="${TOPDIR}/plugins/module_utils"
 
-# Prepare container
-container_id=""
-container_status=("-f" "status=created" "-f" "status=running")
-[ -n "${scenario}" ] && container_id="$(${engine} ps --all -q -f "name=${scenario}" "${container_status[@]}")"
-if [ -z "${container_id}" ]
-then
-    # Retrieve image and start container.
-    log info "Pulling FreeIPA image '${IMAGE_REPO}:${IMAGE_TAG}'..."
-    img_id=$(${engine} pull -q "${IMAGE_REPO}:${IMAGE_TAG}")
-    log info "Creating container..."
-    CONFIG="--hostname ${hostname} --memory ${MEMORY}g --memory-swap -1 --dns none --add-host ipaserver.test.local:127.0.0.1"
-    [ -n "${scenario}" ] && CONFIG="${CONFIG} --name ${scenario}"
-    # shellcheck disable=SC2086
-    container_id=$(${engine} create ${CONFIG} "${img_id}" || die "Cannot create container")
-    echo "CONTAINER: ${container_id}"
-fi
-scenario="${scenario:-$(${engine} ps -q --format "{{.Names}}" --filter "id=${container_id}" "${container_status[@]}")}"
-log debug "Using container: ${scenario}"
-
 # Start container
-make_inventory "${scenario}"
+"${SCRIPTDIR}/setup_test_container.sh" -e "${engine}" -m "${MEMORY}" -p "${ansible_interpreter}" -i "${IMAGE_TAG}" -n "${IPA_HOSTNAME}" -a "${scenario}" || die "Failed to setup test container"
-log info "Starting container for ${scenario}..."
-quiet ${engine} start "${scenario}"
-
-# create /etc/resolve.conf
-run_inline_playbook <<EOF || die "Failed to create /etc/resolv.conf"
----
-- name: Create /etc/resolv.conf
-  hosts: ipaserver
-  gather_facts: no
-  become: yes
-  tasks:
-  - name: Create /etc/resolv.conf
-    ansible.builtin.copy:
-      dest: /etc/resolv.conf
-      mode: 0644
-      content: |
-        search test.local
-        nameserver 127.0.0.1
-...
-EOF
-
-# wait for FreeIPA services to be available
-run_inline_playbook <<EOF || die "Failed to verify IPA or KDC services."
----
-- name: Wait for IPA services to be available
-  hosts: ipaserver
-  gather_facts: no
-  tasks:
-  - name: Wait for IPA to be started.
-    ansible.builtin.systemd:
-      name: ipa
-      state: started
-  - name: Wait for Kerberos KDC to be started.
-    ansible.builtin.systemd:
-      name: krb5kdc
-      state: started
-    register: result
-    until: not result.failed
-    retries: 30
-    delay: 5
-  - name: Check if TGT is available for admin.
-    ansible.builtin.shell:
-      cmd: echo SomeADMINpassword | kinit -c ansible_freeipa_cache admin
-    register: result
-    until: not result.failed
-    retries: 30
-    delay: 5
-  - name: Cleanup TGT.
-    ansible.builtin.shell:
-      cmd: kdestroy -c ansible_freeipa_cache -A
-...
-EOF
-
-# check image software versions.
-run_inline_playbook <<EOF || die "Failed to verify software installation."
----
-- name: Software environment.
-  hosts: ipaserver
-  become: yes
-  gather_facts: no
-  tasks:
-  - name: Retrieve versions.
-    shell:
-      cmd: |
-        rpm -q freeipa-server freeipa-client ipa-server ipa-client 389-ds-base pki-ca krb5-server
-        cat /etc/redhat-release
-        uname -a
-    register: result
-  - name: Testing environment.
-    debug:
-      var: result.stdout_lines
-EOF
 
 
 # run tests
@@ -396,6 +160,9 @@ RESULT=0
 
 export RUN_TESTS_IN_DOCKER=${engine}
 export IPA_SERVER_HOST="${scenario}"
+# Ensure proper ansible_python_interpreter is used by pytest.
+export IPA_PYTHON_PATH="${ansible_interpreter}"
+
 if [ ${SPLITS} -ne 0 ]
 then
     EXTRA_OPTIONS="${EXTRA_OPTIONS} --splits=${SPLITS} --group=${GROUP} --randomly-seed=${SEED}"
@@ -417,11 +184,11 @@ IPA_VERBOSITY="${verbose}"
 [ -n "${IPA_VERBOSITY}" ] && export IPA_VERBOSITY
 
 # shellcheck disable=SC2086
-if ! pytest -m "playbook" --verbose --color=yes ${EXTRA_OPTIONS}
+if ! pytest -m "playbook" --verbose --color=yes --suppress-no-test-exit-code --junit-xml=TEST-results-group-${GROUP:-1}.xml ${EXTRA_OPTIONS}
 then
     RESULT=2
     log error "Container not stopped for verification: ${scenario}"
-    log info "Container: $(podman ps -f "id=${container_id}" --format "{{.Names}} - {{.ID}}")"
+    log info "Container: $(${engine} ps -f "name=${scenario}" --format "{{.Names}} - {{.ID}}")"
 fi
 [ -z "${CONTINUE_ON_ERROR}" ] && [ $RESULT -ne 0 ] && die "Stopping on test failure."
 

utils/set_test_modules

@@ -14,6 +14,8 @@ die() {
 
 TOPDIR="$(dirname "${BASH_SOURCE[0]}")/.."
 
+[ -n "$(command -v python3)" ] && python="$(command -v python3)" || python="$(command -v python2)"
+
 pushd "${TOPDIR}" >/dev/null 2>&1 || die "Failed to change directory."
 
 files_list=$(mktemp)
@@ -25,7 +27,7 @@ git diff "${remote}/master" --name-only > "${files_list}"
 git remote remove ${remote}
 
 # Get all modules that should have tests executed
-enabled_modules="$(python utils/get_test_modules.py $(cat "${files_list}"))"
+enabled_modules="$(${python} utils/get_test_modules.py $(cat "${files_list}"))"
 [ -z "${enabled_modules}" ] && enabled_modules="None"
 
 # Get individual tests that should be executed

utils/setup_test_container.sh

@@ -0,0 +1,119 @@
+#!/bin/bash -eu
+
+SCRIPTDIR="$(readlink -f "$(dirname "$0")")"
+
+# shellcheck source=utils/shcontainer
+. "${SCRIPTDIR}/shcontainer"
+# shellcheck source=utils/shansible
+. "${SCRIPTDIR}/shansible"
+
+usage() {
+    local prog="${0##*/}"
+    cat <<EOF
+usage: ${prog} [-h] [-l] [-a] [-e ENGINE] [-i IMAGE] [-m MEMORY] [-n HOSTNAME] NAME
+    ${prog} starts a container to test ansible-freeipa.
+
+EOF
+}
+
+help() {
+    usage
+    echo -e "$(cat <<EOF
+Arguments:
+
+  NAME            set the container name
+
+Options:
+
+  -h              display this message and exit
+  -l              list available images
+  -a              Test Ansible connection.
+  -e ENGINE       set the container engine to use
+                  (default: ${WHITE}podman${RST}, if available)
+  -i IMAGE        select image to run the tests (default: fedora-latest)
+  -m MEMORY       set container memory, in GiB (default: 3)
+  -n HOSTNAME     set the hostname in the container
+                  (default: ipaserver.test.local)
+  -p INTERPRETER  Python interpreter to use on target container
+EOF
+)"
+}
+
+list_images() {
+    local quay_api="https://quay.io/api/v1/repository/ansible-freeipa/upstream-tests/tag"
+    echo -e "${WHITE}Available images:"
+    curl --silent -L "${quay_api}" | jq '.tags[]|.name' | tr -d '"'| sort | uniq | sed "s/.*/    &/"
+    echo -e "${RST}"
+}
+
+IMAGE_TAG="fedora-latest"
+MEMORY="${MEMORY:-3}"
+IPA_HOSTNAME="${IPA_HOSTNAME:-"ipaserver.test.local"}"
+test_env="${test_env:-"/tmp"}"
+ansible_interpreter="/usr/bin/python3"
+engine="podman"
+ansible_test=""
+
+while getopts ":hae:i:lm:n:p:" option
+do
+    case "$option" in
+        h) help && exit 0 ;;
+        a) ansible_test="yes" ;;
+        e) engine="${OPTARG}" ;;
+        i) IMAGE_TAG="${OPTARG}" ;;
+        l) list_images && exit 0 || exit 1;;
+        m) MEMORY="${OPTARG}" ;;
+        n) IPA_HOSTNAME="${OPTARG}" ;;
+        p) ansible_interpreter="${OPTARG}" ;;
+        *) die -u "Invalid option: ${OPTARG}" ;;
+    esac
+done
+
+export IPA_HOSTNAME MEMORY IMAGE_TAG scenario
+
+shift $((OPTIND - 1))
+[ $# == 1 ] || die -u "You must provide the name for a single container." 
+scenario="${1}"
+shift
+
+prepare_container "${scenario}" "${IMAGE_TAG}"
+start_container "${scenario}"
+
+# wait for FreeIPA services to be available (usually ~45 seconds)
+log info "Wait for container to be initialized."
+wait=15
+while podman exec "${scenario}" systemctl list-jobs | grep -qvi "no jobs running"
+do
+    log none "Waiting ${wait}s... "
+    sleep "${wait}"
+    log none "Retrying".
+done
+
+# run tests
+
+# ensure we can get a TGT for admin
+log info "Testing kinit with admin."
+# shellcheck disable=SC2016
+"${engine}" exec "${scenario}" /bin/sh -c 'for i in $(seq 5); do echo "SomeADMINpassword" | kinit -c ansible_freeipa_cache admin && kdestroy -c ansible_freeipa_cache -A && break; echo "Failed to get TGT. Retrying in 10s..."; sleep 10; done' || die "Failed to grant admin TGT."
+
+# shellcheck disable=SC2154
+log info "Creating inventory."
+make_inventory "${scenario}" "${engine}" "${ansible_interpreter:-"/usr/bin/python3"}"
+if [ -z "${inventory:-''}" ]
+then
+    log error "Could not create inventory file."
+else
+    # shellcheck disable=SC2154
+    log info "Inventory path: [${inventory}]"
+    # shellcheck disable=SC2154
+    log debug "$(cat "${inventory}")"
+    if [ "${ansible_test}" == "yes" ]
+    then
+        log info "Testing Ansible connection."
+        # shellcheck disable=SC2154
+        run_if_exists ansible_ping "${inventory}"
+        log info "Querying installed software"
+        run_if_exists query_container_installed_software
+    fi
+fi
+

utils/shansible

@@ -0,0 +1,88 @@
+#!/bin/bash -eu
+# This file is meant to be source'd by other scripts
+
+SCRIPTDIR="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+# shellcheck source=utils/shfun
+. "${SCRIPTDIR}/shfun"
+
+install_ansible() {
+    ANSIBLE_VERSION="${1:-${ANSIBLE_VERSION:-"ansible-core"}}"
+    [ $# -gt 0 ] && shift
+    log info "Installing Ansible: ${ANSIBLE_VERSION}"
+    pip install --quiet "${ANSIBLE_VERSION}"
+    log debug "Ansible version: $(ansible --version | sed -n "1p")${RST}"
+
+    if [ -n "${ANSIBLE_COLLECTIONS}" ]
+    then
+        log warn "Installed collections will not be removed after execution."
+        log none "Installing: Ansible Collection ${ANSIBLE_COLLECTIONS}"
+        # shellcheck disable=SC2086
+        quiet ansible-galaxy collection install ${ANSIBLE_COLLECTIONS} || die "Failed to install Ansible collections."
+    fi
+    export ANSIBLE_VERSION
+}
+
+run_inline_playbook() {
+    local playbookdir playbook err
+    playbookdir=${1:-"playbooks"}
+    quiet mkdir -p "${playbookdir}"
+    playbook=$(mktemp "${playbookdir}/ansible-freeipa-test-playbook_ipa.XXXXXXXX" 2>/dev/null)
+    # In some configurations, it may not be possible to use another
+    # directory, so we store the playbook in the current one.
+    # [ -z "${playbook}" ] && playbook=$(mktemp "ansible-freeipa-test-playbook_ipa.XXXXXXXX")
+
+    inventory="${inventory:-${test_env:-"."}/inventory}"
+    quiet mkdir -p "${playbookdir}"
+    cat - >"${playbook}"
+    # shellcheck disable=SC2086
+    run_if_exists ansible-playbook ${ansible_options:-} -i "${inventory}" "${playbook}"
+    err=$?
+    rm -f "${playbook}"
+    return ${err}
+}
+
+make_inventory() {
+    local scenario pod_engine ansible_interpreter
+    scenario=$1
+    pod_engine="${engine:-${2:-podman}}"
+    ansible_interpreter="${3:-${ansible_interpreter:-"/usr/bin/python3"}}"
+    export inventory="${test_env:-"."}/inventory"
+    log info "Inventory file: ${inventory}"
+    cat << EOF > "${inventory}"
+[ipaserver]
+${scenario} ansible_connection=${pod_engine} ansible_python_interpreter=${ansible_interpreter}
+[ipaserver:vars]
+ipaserver_domain = test.local
+ipaserver_realm = TEST.LOCAL
+EOF
+}
+
+query_container_installed_software() {
+    # check image software versions.
+    run_inline_playbook "${test_env:-"/tmp"}/playbooks" <<EOF || die "Failed to verify software installation."
+---
+- name: Software environment.
+  hosts: ipaserver
+  become: yes
+  gather_facts: no
+  tasks:
+  - name: Retrieve versions.
+    ansible.builtin.shell: |
+      cat /etc/redhat-release
+      ${python:-"python3"} --version
+      rpm -q freeipa-server freeipa-client ipa-server ipa-client 389-ds-base pki-ca krb5-server
+      uname -a
+    register: result
+  - name: Testing environment.
+    ansible.builtin.debug:
+      var: result.stdout_lines
+EOF
+}
+
+ansible_ping() {
+    # shellcheck disable=SC2086
+    ansible ${ansible_options:-} -m ping -i "${1:-${inventory}}" all || die "Could not connect to container."
+}
+
+export ANSIBLE_VERSION=${ANSIBLE_VERSION:-'ansible-core'}

utils/shcontainer

@@ -0,0 +1,61 @@
+#!/bin/bash -eu
+# This file is meant to be source'd by other scripts
+
+SCRIPTDIR="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+IMAGE_REPO="quay.io/ansible-freeipa/upstream-tests"
+
+# shellcheck source=utils/shfun
+. "${SCRIPTDIR}/shfun"
+
+stop_container() {
+    local scenario=${1}
+    log none "Stopping container..."
+    quiet "${engine}" stop "${scenario}"
+    log none "Removing container..."
+    quiet "${engine}" rm "${scenario}"
+}
+
+# Prepare container
+prepare_container() {
+    local container_id container_status hostname scenario
+    local IMAGE_TAG img_id CONFIG
+    container_id=""
+    container_status=("-f" "status=created" "-f" "status=running")
+    hostname="${IPA_HOSTNAME:-"ipaserver.test.local"}"
+    scenario="${1:-${scenario:-"freeipa-tests"}}"
+    IMAGE_TAG="${2:-${IMAGE_TAG:-fedora-latest}}"
+    [ -n "${scenario}" ] && container_id="$(${engine} ps --all -q -f "name=${scenario}" "${container_status[@]}")"
+    if [ -z "${container_id}" ]
+    then
+        # Retrieve image and start container.
+        log info "Pulling FreeIPA image '${IMAGE_REPO}:${IMAGE_TAG}'..."
+        img_id=$(${engine} pull -q "${IMAGE_REPO}:${IMAGE_TAG}")
+        log debug "Hostname: ${hostname}"
+        log info "Creating container..."
+        CONFIG="--systemd true --hostname ${hostname} --memory ${MEMORY}g --memory-swap -1 --no-hosts"
+        [ -n "${scenario}" ] && CONFIG="${CONFIG} --name ${scenario}"
+        # shellcheck disable=SC2086
+        container_id=$(${engine} create ${CONFIG} "${img_id}" || die "Cannot create container")
+        log none "CONTAINER: ${container_id}"
+    fi
+    export scenario="${scenario:-$(${engine} ps -q --format "{{.Names}}" --filter "id=${container_id}" "${container_status[@]}")}"
+    log debug "Prepared container: ${scenario}"
+}
+
+start_container() {
+    local scenario="${1:-${scenario}}"
+    log info "Starting container for ${scenario}..."
+    "${engine}" start "${scenario}"
+}
+
+if [ -z "$(command -v podman)" ]
+then
+    engine="docker"
+    engine_collection="community.docker"
+else
+    engine="podman"
+    engine_collection="containers.podman"
+fi
+
+export engine engine_collection