From 68bca84481384c7d2aa5b695d1751a953e3ef44e Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Wed, 26 Jun 2024 14:46:53 +0000 Subject: [PATCH 1/5] utils: Rewrite run-tests.sh to use functions and extenal scripts To modify Azure tests and depend on shell scripts and pytest instead of molecule, the run-tests.sh script has been rewritten to depend on bash functions and on a bash script that prepare and start a testing container. This patch adds a new script, 'utils/setup_test_container.sh' that can be used to start a new container, using either podman or docker, based on the available ansible-freeipa images. The new container can then be used to run ansible-freeipa tests against it. Also the following files with bash functions were added, and are used by both scripts: utils/shansible: Functions to run playbooks in the container utils/shcontainer: Functions to setup/run a container utils/shfun: Generic shell helper functions (e.g.: log) --- utils/run-tests.sh | 365 ++++++---------------------------- utils/set_test_modules | 4 +- utils/setup_test_container.sh | 119 +++++++++++ utils/shansible | 88 ++++++++ utils/shcontainer | 61 ++++++ 5 files changed, 337 insertions(+), 300 deletions(-) create mode 100755 utils/setup_test_container.sh create mode 100644 utils/shansible create mode 100644 utils/shcontainer diff --git a/utils/run-tests.sh b/utils/run-tests.sh index e814aed0..140998be 100755 --- a/utils/run-tests.sh +++ b/utils/run-tests.sh @@ -1,38 +1,19 @@ #!/bin/bash -eu -trap interrupt_exception SIGINT +SCRIPTDIR="$(readlink -f "$(dirname "$0")")" +TOPDIR="$(readlink -f "${SCRIPTDIR}/..")" -RST="\033[0m" -RED="\033[31m" -# BRIGHTRED="\033[31;1m" -# GREEN="\033[32m" -BRIGHTGREEN="\033[32;1m" -# BROWN="\033[33m" -YELLOW="\033[33;1m" -# NAVY="\033[34m" -BLUE="\033[34;1m" -# MAGENTA="\033[35m" -# BRIGHTMAGENTA="\033[35;1m" -# DARKCYAN="\033[36m" -# CYAN="\033[36;1m" -# BLACK="\033[30m" -# DARKGRAY="\033[30;1m" -# GRAY="\033[37m" -WHITE="\033[37;1m" - -TOPDIR="$(readlink -f "$(dirname "$0")/..")" - -interrupt_exception() { - trap - SIGINT - log warn "User interrupted test execution." - cleanup - exit 1 -} +# shellcheck source=utils/shfun +. "${SCRIPTDIR}/shfun" +# shellcheck source=utils/shcontainer +. "${SCRIPTDIR}/shcontainer" +# shellcheck source=utils/shansible +. "${SCRIPTDIR}/shansible" usage() { local prog="${0##*/}" cat </dev/null 2>&1 -} - -in_python_virtualenv() { - local script - read -r -d "" script <"${playbook}" - ansible-playbook -i "${inventory}" "${playbook}" - err=$? - rm "${playbook}" - return ${err} -} - -die() { - usg="N" - if [ "${1}" == "-u" ] - then - usg="Y" - shift 1 - fi - log error "${*}" - STOP_CONTAINER="N" - cleanup - [ "${usg}" == "Y" ] && usage - exit 1 -} - -make_inventory() { - local scenario=$1 engine=${2:-podman} - inventory="${test_env}/inventory" - log info "Inventory file: ${inventory}" - cat << EOF > "${inventory}" -[ipaserver] -${scenario} ansible_connection=${engine} -[ipaserver:vars] -ipaserver_domain = test.local -ipaserver_realm = TEST.LOCAL -EOF -} - -stop_container() { - local scenario=${1} engine=${2:-podman} - echo "Stopping container..." - quiet "${engine}" stop "${scenario}" - echo "Removing container..." - quiet "${engine}" rm "${scenario}" -} - -cleanup() { - if [ $# -gt 0 ] - then - if [ "${STOP_CONTAINER}" != "N" ] - then - stop_container "${1}" "${2}" - rm "${inventory}" - else - log info "Keeping container: $(podman ps --format "{{.Names}} - {{.ID}}" --filter "name=${1}")" - fi - fi - if [ "${STOP_VIRTUALENV}" == "Y" ] - then - echo "Deactivating virtual environment" - deactivate - fi -} - -list_images() { - local quay_api="https://quay.io/api/v1/repository/ansible-freeipa/upstream-tests/tag" - echo -e "${WHITE}Available images:" - curl --silent -L "${quay_api}" | jq '.tags[]|.name' | tr -d '"'| sort | uniq | sed "s/.*/ &/" - echo -e "${RST}" -} # Defaults - -ANSIBLE_VERSION=${ANSIBLE_VERSION:-'ansible-core'} verbose="" -FORCE_ENV="N" +engine="${engine:-"podman"}" CONTINUE_ON_ERROR="" STOP_CONTAINER="Y" STOP_VIRTUALENV="N" declare -a ENABLED_MODULES declare -a ENABLED_TESTS -ENABLED_MODULES=() -ENABLED_TESTS=() -test_env="${TESTENV_DIR:-${VIRTUAL_ENV:-/tmp/ansible-freeipa-tests}}" -engine="podman" -IMAGE_REPO="quay.io/ansible-freeipa/upstream-tests" +read -r -a ENABLED_MODULES <<< "${IPA_ENABLED_MODULES:-""}" +read -r -a ENABLED_TESTS <<< "${IPA_ENABLED_MODULES:-""}" IMAGE_TAG="fedora-latest" -scenario="" +scenario="freeipa-tests" MEMORY=3 -hostname="ipaserver.test.local" -SEED="" -GROUP=0 +IPA_HOSTNAME="ipaserver.test.local" +SEED="$(date "+%Y%m%d")" +GROUP=1 SPLITS=0 -ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"containers.podman"} - +ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"${engine_collection}"} +SKIP_ANSIBLE="" +ansible_interpreter="/usr/bin/python3" EXTRA_OPTIONS="" +unset ANSIBLE_VERSION # Process command options -while getopts ":hA:c:ei:Klms:vx" option +while getopts ":ha:Ac:ei:Klm:p:s:S:vx" option do case "$option" in h) help && exit 0 ;; A) - [ ${#ENABLED_MODULES[@]} -eq 0 ] || die -u "Can't use '-A' with '-s'" - SEED="$(cut -d. -f1 <<< "${OPTARG}" | tr -d "-")" - GROUP="$(cut -d. -f2 <<< "${OPTARG}")" - if [ -z "${SEED}" ] || [ -z "${GROUP}" ] - then - die -u "Seed for '-A' must have the format YYYYMMDD.N" - fi - SPLITS=3 - ;; + [ -n "${ANSIBLE_VERSION:-""}" ] && die "Can't use -A with '-a'" + SKIP_ANSIBLE="YES" + ;; + a) + [ "${SKIP_ANSIBLE:-"no"}" == "YES" ] && die "Can't use -A with '-a'" + ANSIBLE_VERSION="${OPTARG}" + ;; c) scenario="${OPTARG}" ;; e) FORCE_ENV="Y" ;; i) IMAGE_TAG="${OPTARG}" ;; K) STOP_CONTAINER="N" ;; - l) list_images && exit 0 || exit 1;; + l) "${SCRIPTDIR}"/setup_test_container.sh -l && exit 0 || exit 1 ;; m) MEMORY="${OPTARG}" ;; + p) ansible_interpreter="${OPTARG}" ;; s) - [ ${SPLITS} -ne 0 ] && die -u "Can't use '-A' with '-s'" + [ ${SPLITS} -ne 0 ] && die -u "Can't use '-S' with '-s'" if [ -d "${TOPDIR}/tests/${OPTARG}" ] then ENABLED_MODULES+=("${OPTARG}") @@ -220,6 +101,16 @@ do log error "Invalid suite: ${OPTARG}" fi ;; + S) + [ ${#ENABLED_MODULES[@]} -eq 0 ] || die -u "Can't use '-A' with '-s'" + SEED="$(cut -d. -f1 <<< "${OPTARG}" | tr -d "-")" + GROUP="$(cut -d. -f2 <<< "${OPTARG}")" + if [ -z "${SEED}" ] || [ -z "${GROUP}" ] + then + die -u "Seed for '-A' must have the format YYYYMMDD.N" + fi + SPLITS=3 + ;; v) verbose=${verbose:--}${option} ;; x) EXTRA_OPTIONS="$EXTRA_OPTIONS --exitfirst" ;; *) die -u "Invalid option: ${OPTARG}" ;; @@ -240,155 +131,28 @@ done [ ${SPLITS} -eq 0 ] && [ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined." +export STOP_CONTAINER FORCE_ENV STOP_VIRTUALENV ansible_interpreter + +# Ensure $python is set +[ -z "${python}" ] && python="python3" + +log info "Controller Python executable: ${python}" +${python} --version + # Prepare virtual environment -VENV=$(in_python_virtualenv && echo Y || echo N) +start_virtual_environment +log info "Installing dependencies from 'requirements-tests.txt'" +pip install --upgrade -r "${TOPDIR}/requirements-tests.txt" -if [ "${FORCE_ENV}" == "Y" ] -then - [ "${VENV}" == "Y" ] && deactivate - VENV="N" - rm -rf "$test_env" - log info "Virtual environment will be (re)created." -fi - -if [ "$VENV" == "N" ] -then - log info "Preparing virtual environment: ${test_env}" - if [ ! -d "${test_env}" ] - then - log info "Creating virtual environment: ${test_env}..." - if ! python3 -m venv "${test_env}" - then - die "Cannot create virtual environment." - fi - fi - if [ -f "${test_env}/bin/activate" ] - then - log info "Starting virtual environment: ${test_env}" - # shellcheck disable=SC1091 - . "${test_env}/bin/activate" || die "Cannot activate environment." - STOP_VIRTUALENV="Y" - else - die "Cannot activate environment." - fi - log info "Installing required tools." - log none "Upgrading: pip setuptools wheel" - pip install --quiet --upgrade pip setuptools wheel - log info "Installing dependencies from 'requirements-tests.txt'" - pip install --quiet -r "${TOPDIR}/requirements-tests.txt" - log info "Installing Ansible: ${ANSIBLE_VERSION}" - pip install --quiet "${ANSIBLE_VERSION}" - log debug "Ansible version: $(ansible --version | sed -n "1p")${RST}" -else - log info "Using current virtual environment." -fi - -if [ -n "${ANSIBLE_COLLECTIONS}" ] -then - log warn "Installed collections will not be removed after execution." - log none "Installing: Ansible Collection ${ANSIBLE_COLLECTIONS}" - # shellcheck disable=SC2086 - quiet ansible-galaxy collection install ${ANSIBLE_COLLECTIONS} || die "Failed to install Ansible collections." -fi +[ -z "${SKIP_ANSIBLE}" ] && install_ansible "${ANSIBLE_VERSION:-"ansible-core"}" # Ansible configuration export ANSIBLE_ROLES_PATH="${TOPDIR}/roles" -export ANSIBLE_LIBRARY="${TOPDIR}/plugins:${TOPDIR}/molecule" +export ANSIBLE_LIBRARY="${TOPDIR}/plugins" export ANSIBLE_MODULE_UTILS="${TOPDIR}/plugins/module_utils" -# Prepare container -container_id="" -container_status=("-f" "status=created" "-f" "status=running") -[ -n "${scenario}" ] && container_id="$(${engine} ps --all -q -f "name=${scenario}" "${container_status[@]}")" -if [ -z "${container_id}" ] -then - # Retrieve image and start container. - log info "Pulling FreeIPA image '${IMAGE_REPO}:${IMAGE_TAG}'..." - img_id=$(${engine} pull -q "${IMAGE_REPO}:${IMAGE_TAG}") - log info "Creating container..." - CONFIG="--hostname ${hostname} --memory ${MEMORY}g --memory-swap -1 --dns none --add-host ipaserver.test.local:127.0.0.1" - [ -n "${scenario}" ] && CONFIG="${CONFIG} --name ${scenario}" - # shellcheck disable=SC2086 - container_id=$(${engine} create ${CONFIG} "${img_id}" || die "Cannot create container") - echo "CONTAINER: ${container_id}" -fi -scenario="${scenario:-$(${engine} ps -q --format "{{.Names}}" --filter "id=${container_id}" "${container_status[@]}")}" -log debug "Using container: ${scenario}" - # Start container -make_inventory "${scenario}" -log info "Starting container for ${scenario}..." -quiet ${engine} start "${scenario}" - -# create /etc/resolve.conf -run_inline_playbook </dev/null 2>&1 || die "Failed to change directory." files_list=$(mktemp) @@ -25,7 +27,7 @@ git diff "${remote}/master" --name-only > "${files_list}" git remote remove ${remote} # Get all modules that should have tests executed -enabled_modules="$(python utils/get_test_modules.py $(cat "${files_list}"))" +enabled_modules="$(${python} utils/get_test_modules.py $(cat "${files_list}"))" [ -z "${enabled_modules}" ] && enabled_modules="None" # Get individual tests that should be executed diff --git a/utils/setup_test_container.sh b/utils/setup_test_container.sh new file mode 100755 index 00000000..0916c27f --- /dev/null +++ b/utils/setup_test_container.sh @@ -0,0 +1,119 @@ +#!/bin/bash -eu + +SCRIPTDIR="$(readlink -f "$(dirname "$0")")" + +# shellcheck source=utils/shcontainer +. "${SCRIPTDIR}/shcontainer" +# shellcheck source=utils/shansible +. "${SCRIPTDIR}/shansible" + +usage() { + local prog="${0##*/}" + cat </dev/null) + # In some configurations, it may not be possible to use another + # directory, so we store the playbook in the current one. + # [ -z "${playbook}" ] && playbook=$(mktemp "ansible-freeipa-test-playbook_ipa.XXXXXXXX") + + inventory="${inventory:-${test_env:-"."}/inventory}" + quiet mkdir -p "${playbookdir}" + cat - >"${playbook}" + # shellcheck disable=SC2086 + run_if_exists ansible-playbook ${ansible_options:-} -i "${inventory}" "${playbook}" + err=$? + rm -f "${playbook}" + return ${err} +} + +make_inventory() { + local scenario pod_engine ansible_interpreter + scenario=$1 + pod_engine="${engine:-${2:-podman}}" + ansible_interpreter="${3:-${ansible_interpreter:-"/usr/bin/python3"}}" + export inventory="${test_env:-"."}/inventory" + log info "Inventory file: ${inventory}" + cat << EOF > "${inventory}" +[ipaserver] +${scenario} ansible_connection=${pod_engine} ansible_python_interpreter=${ansible_interpreter} +[ipaserver:vars] +ipaserver_domain = test.local +ipaserver_realm = TEST.LOCAL +EOF +} + +query_container_installed_software() { + # check image software versions. + run_inline_playbook "${test_env:-"/tmp"}/playbooks" < Date: Tue, 2 Jul 2024 02:24:17 +0000 Subject: [PATCH 2/5] tests: Allow to set Python interpreter to be used by Ansible On some systems it is required or desired to run Ansible with a specific Python interpreter. This patch allows the selection of the Python binary to use for the pytest playbook tests by setting the environment variable IPA_PYTHON_PATH. Set it the the full path of the Python interpreter. --- tests/README.md | 6 ++++++ tests/utils.py | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/README.md b/tests/README.md index ed54b0b8..1e9d4e4a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -36,6 +36,12 @@ environment variable. For example: IPA_SSH_PASSWORD= IPA_SERVER_HOST= pytest ``` +If you want, or need to to set the Python interpreter to use, you must set `IPA_PYTHON_PATH` +environment variable. For example: + +``` +IPA_PYTHON_PATH=/usr/bin/python3.14 IPA_SERVER_HOST= pytest +``` To run a single test use the full path with the following format: diff --git a/tests/utils.py b/tests/utils.py index 01991aaa..89a24fc4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -43,6 +43,10 @@ def get_ssh_password(): return os.getenv("IPA_SSH_PASSWORD") +def get_python_interpreter(): + return os.getenv("IPA_PYTHON_PATH") + + def get_server_host(): return os.getenv("IPA_SERVER_HOST") @@ -97,6 +101,12 @@ def get_inventory_content(): if sshpass: ipa_server_host += " ansible_ssh_pass=%s" % sshpass + python_interpreter = get_python_interpreter() + if python_interpreter: + ipa_server_host += ( + " ansible_python_interpreter=%s" % python_interpreter + ) + lines = [ "[ipaserver]", ipa_server_host, From 94311f439c2f7bac18dc0bafa8f18a87cc988eb3 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Sat, 29 Jun 2024 21:20:33 +0000 Subject: [PATCH 3/5] gitignore: Ignore test results from run-tests.sh --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d74bada6..626d54fa 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ importer_result.json /.venv/ tests/logs/ +TEST*.xml From 8515c9a48bad1b117528c8f4db6af7ecf8ccb5e5 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Wed, 26 Jun 2024 14:55:53 +0000 Subject: [PATCH 4/5] upstream ci: Remove 'molecule' from tests. This patch removes 'molecule' as a dependency for tests, by using the scripts under `utils` to setup the environment. By not using molecule, we have more flexibility on using either docker or podman as the container engine, and makes it easy to reproduce the environment on different distros, allowing for a more consistent error reproduction off Azure. --- tests/azure/pr-pipeline.yml | 42 ++++++++------- tests/azure/templates/fast_tests.yml | 10 ++-- tests/azure/templates/playbook_fast.yml | 68 +++++++++++-------------- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/tests/azure/pr-pipeline.yml b/tests/azure/pr-pipeline.yml index 2345347d..40d78a9b 100644 --- a/tests/azure/pr-pipeline.yml +++ b/tests/azure/pr-pipeline.yml @@ -15,8 +15,8 @@ stages: - template: templates/fast_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest - ansible_version: "-core >=2.14,<2.15" + distro: fedora-latest + ansible_version: "-core >=2.15,<2.16" # Galaxy on Fedora @@ -26,8 +26,8 @@ stages: - template: templates/fast_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest - ansible_version: "-core >=2.14,<2.15" + distro: fedora-latest + ansible_version: "-core >=2.15,<2.16" # CentOS 9 Stream @@ -37,8 +37,8 @@ stages: - template: templates/fast_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c9s - ansible_version: "-core >=2.14,<2.15" + distro: c9s + ansible_version: "-core >=2.15,<2.16" # CentOS 8 Stream @@ -48,19 +48,23 @@ stages: - template: templates/fast_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c8s - ansible_version: "-core >=2.14,<2.15" + distro: c8s + ansible_version: "-core >=2.15,<2.16" + target_python: "/usr/libexec/platform-python" +# CentOS 7 cannot be used with current systemd +# # CentOS 7 - -- stage: CentOS_7 - dependsOn: [] - jobs: - - template: templates/fast_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: centos-7 - ansible_version: "-core >=2.14,<2.15" +# +# - stage: CentOS_7 +# dependsOn: [] +# jobs: +# - template: templates/fast_tests.yml +# parameters: +# build_number: $(Build.BuildNumber) +# distro: centos-7 +# ansible_version: "-core >=2.15,<2.16" +# target_python: "/usr/bin/python2" # Rawhide @@ -70,5 +74,5 @@ stages: - template: templates/fast_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-rawhide - ansible_version: "-core >=2.14,<2.15" + distro: fedora-rawhide + ansible_version: "-core >=2.15,<2.16" diff --git a/tests/azure/templates/fast_tests.yml b/tests/azure/templates/fast_tests.yml index fdb1ea0d..ac26ce8f 100644 --- a/tests/azure/templates/fast_tests.yml +++ b/tests/azure/templates/fast_tests.yml @@ -1,6 +1,6 @@ --- parameters: - - name: scenario + - name: distro type: string default: fedora-latest - name: build_number @@ -8,6 +8,9 @@ parameters: - name: ansible_version type: string default: "" + - name: target_python + type: string + default: "/usr/bin/python3" jobs: - template: playbook_fast.yml @@ -15,13 +18,14 @@ jobs: group_number: 1 number_of_groups: 1 build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} + distro: ${{ parameters.distro }} ansible_version: ${{ parameters.ansible_version }} python_version: '< 3.12' + target_python: ${{ parameters.target_python }} # - template: pytest_tests.yml # parameters: # build_number: ${{ parameters.build_number }} -# scenario: ${{ parameters.scenario }} +# distro: ${{ parameters.distro }} # ansible_version: ${{ parameters.ansible_version }} # python_version: '< 3.12' diff --git a/tests/azure/templates/playbook_fast.yml b/tests/azure/templates/playbook_fast.yml index a01d2c34..a00a7875 100644 --- a/tests/azure/templates/playbook_fast.yml +++ b/tests/azure/templates/playbook_fast.yml @@ -6,7 +6,7 @@ parameters: - name: number_of_groups type: number default: 1 - - name: scenario + - name: distro type: string default: fedora-latest - name: ansible_version @@ -17,28 +17,28 @@ parameters: default: 3.x - name: build_number type: string + - name: target_python + type: string + default: "/usr/bin/python3" jobs: - job: Test_Group${{ parameters.group_number }} - displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) + displayName: Run playbook tests ${{ parameters.distro }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) timeoutInMinutes: 360 variables: - template: variables.yaml - - template: variables_${{ parameters.scenario }}.yaml + - template: variables_${{ parameters.distro }}.yaml steps: - task: UsePythonVersion@0 inputs: versionSpec: '${{ parameters.python_version }}' - script: | - pip install \ - "molecule-plugins[docker]" \ - "requests<2.29" \ - "ansible${{ parameters.ansible_version }}" + pip install "ansible${{ parameters.ansible_version }}" retryCountOnTaskFailure: 5 - displayName: Install molecule and Ansible + displayName: Install Ansible - - script: ansible-galaxy collection install community.docker ansible.posix + - script: ansible-galaxy collection install containers.podman retryCountOnTaskFailure: 5 displayName: Install Ansible collections @@ -47,43 +47,35 @@ jobs: displayName: Install dependencies - script: | - rm -rf ~/ansible - mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils - cp -a roles/* ~/.ansible/roles - cp -a plugins/modules/* ~/.ansible/library - cp -a plugins/module_utils/* ~/.ansible/module_utils - molecule create -s ${{ parameters.scenario }} - retryCountOnTaskFailure: 5 - displayName: Setup test container - env: - ANSIBLE_LIBRARY: ./molecule + . utils/set_test_modules + python3 utils/check_test_configuration.py ${{ parameters.distro }} + displayName: Check test configuration + + - script: | + utils/setup_test_container.sh \ + -e podman \ + -a \ + -m 4 \ + -n "ipaserver.test.local" \ + -p ${{ parameters.target_python }} \ + -i ${{ parameters.distro }}-server \ + ${{ parameters.distro }}-test + displayName: Setup target container - script: | . utils/set_test_modules - python utils/check_test_configuration.py ${{ parameters.scenario }} - displayName: Check scenario test configuration - - - script: | - . utils/set_test_modules - if ! pytest \ - -m "playbook" \ - --verbose \ - --color=yes \ - --suppress-no-test-exit-code \ - --splits=${{ parameters.number_of_groups }} \ - --group=${{ parameters.group_number }} \ - --randomly-seed=$(date "+%Y%m%d") \ - --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml - then - [ $? -eq 5 ] && true || false - fi + pytest -m "playbook" --verbose --color=yes --suppress-no-test-exit-code --junit-xml=TEST-results-pr-check.xml displayName: Run playbook tests env: - IPA_SERVER_HOST: ${{ parameters.scenario }} - RUN_TESTS_IN_DOCKER: true + ANSIBLE_ROLES_PATH: "${PWD}/roles" + ANSIBLE_LIBRARY: "${PWD}/plugins" + ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils" + IPA_SERVER_HOST: ${{ parameters.distro }}-test + RUN_TESTS_IN_DOCKER: podman IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} IPA_VERBOSITY: "-vvv" + IPA_PYTHON_PATH: ${{ parameters.target_python }} - task: PublishTestResults@2 inputs: From 02ba890eb4ec21e7454a96556741cd8d7ba2f3d9 Mon Sep 17 00:00:00 2001 From: Thomas Woerner Date: Thu, 1 Aug 2024 12:04:22 +0200 Subject: [PATCH 5/5] tests/utils.py: Shorten run_playbook for smaller traceback with assert Most of the content has been moved to the new function _run_playbook to reduce the traceback output in the case of a test failure. --- tests/utils.py | 117 ++++++++++++++++++++++++------------------------- 1 file changed, 57 insertions(+), 60 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 89a24fc4..5d8a806c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -148,30 +148,6 @@ def write_logs(result, test_name): log_file.write(result.stderr.decode("utf-8")) -def _run_playbook(playbook): - """ - Create a inventory using a temporary file and run ansible using it. - - The logs of the run will be placed in `tests/logs/`. - """ - with tempfile.NamedTemporaryFile() as inventory_file: - inventory_file.write(get_inventory_content()) - inventory_file.flush() - cmd_options = ["-i", inventory_file.name] - verbose = os.environ.get("IPA_VERBOSITY", None) - if verbose is not None: - cmd_options.append(verbose) - cmd = ["ansible-playbook"] + cmd_options + [playbook] - # pylint: disable=subprocess-run-check - process = subprocess.run( - cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - test_name = get_test_name_from_playbook_path(playbook) - write_logs(process, test_name) - - return process - - def _truncate(lines, charcount, minlines=0): output = "" line_count = 1 @@ -188,12 +164,11 @@ def _truncate(lines, charcount, minlines=0): return output -def run_playbook(playbook, allow_failures=False): +def _run_playbook(playbook): """ - Run an Ansible playbook and assert the return code. + Create a inventory using a temporary file and run ansible using it. - Call ansible (using _run_playbook function) and assert the result of - the execution. + The logs of the run will be placed in `tests/logs/`. In case of failure the tail of the error message will be displayed as an assertion message. @@ -201,43 +176,65 @@ def run_playbook(playbook, allow_failures=False): The full log of the execution will be available in the directory `tests/logs/`. """ - result = _run_playbook(playbook) + with tempfile.NamedTemporaryFile() as inventory_file: + inventory_file.write(get_inventory_content()) + inventory_file.flush() + cmd_options = ["-i", inventory_file.name] + verbose = os.environ.get("IPA_VERBOSITY", None) + if verbose is not None: + cmd_options.append(verbose) + cmd = ["ansible-playbook"] + cmd_options + [playbook] + process = subprocess.run( + cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False + ) + test_name = get_test_name_from_playbook_path(playbook) + write_logs(process, test_name) - if allow_failures: - return result + msg = "" + if process.returncode != 0: + status_code_msg = "ansible-playbook return code: {0}".format( + process.returncode + ) - status_code_msg = "ansible-playbook return code: {0}".format( - result.returncode - ) - _stdout = result.stdout.decode("utf8") - _stderr = result.stderr.decode("utf8") - # Truncate stdout and stderr in the way that it hopefully - # shows all important information. At least 15 lines of stdout - # (Ansible tasks) and remaining from stderr to fill up to - # maxlen size. - maxlen = 2000 - factor = maxlen / (len(_stdout) + len(_stderr)) - stdout = _truncate(_stdout.splitlines(), - int(factor * len(_stdout)), - minlines=15) - stderr = _truncate(_stderr.splitlines(), maxlen - len(stdout)) + _stdout = process.stdout.decode("utf8") + _stderr = process.stderr.decode("utf8") + # Truncate stdout and stderr in the way that it hopefully + # shows all important information. At least 15 lines of stdout + # (Ansible tasks) and remaining from stderr to fill up to + # maxlen size. + maxlen = 2000 + factor = maxlen / (len(_stdout) + len(_stderr)) + stdout = _truncate(_stdout.splitlines(), + int(factor * len(_stdout)), + minlines=15) + stderr = _truncate(_stderr.splitlines(), maxlen - len(stdout)) - assert_msg = "\n".join( - [ - "", - "-" * 30 + " Captured stdout " + "-" * 30, - stdout, - "-" * 30 + " Captured stderr " + "-" * 30, - stderr, - "-" * 30 + " Playbook Return Code " + "-" * 30, - status_code_msg, - ] - ) + msg = "\n".join( + [ + "", + "-" * 30 + " Captured stdout " + "-" * 30, + stdout, + "-" * 30 + " Captured stderr " + "-" * 30, + stderr + ] + ) + msg += "-" * 30 + " Playbook Return Code " + "-" * 30 + "\n" + msg += status_code_msg - # Need to get the last bytes of msg otherwise Azure - # will cut it out. - assert result.returncode == 0, assert_msg[-2500:] + return process, msg + +def run_playbook(playbook, allow_failures=False): + """ + Run an Ansible playbook and assert the return code. + + Call ansible (using _run_playbook function) and assert the result of + the execution. + """ + result, assert_msg = _run_playbook(playbook) + if not allow_failures: + assert result.returncode == 0, assert_msg return result