From 7e6e6c2dc21534a38764473611f7d99070a043ae Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Tue, 28 Jun 2022 18:29:52 -0300 Subject: [PATCH] run-tests: Run tests locally with upstream CI images This patch allows local execution of playbook tests using ustream CI testing images. Either 'podman' or 'docker' can be used to execute the tests. --- .ansible-lint | 2 + requirements-docker.yml | 3 + requirements-podman.yml | 3 + tests/README.md | 29 +++ tests/utils.py | 30 +-- utils/run-tests.sh | 400 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 453 insertions(+), 14 deletions(-) create mode 100644 requirements-docker.yml create mode 100644 requirements-podman.yml create mode 100755 utils/run-tests.sh diff --git a/.ansible-lint b/.ansible-lint index 9a4af927..7aeb2ef5 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -10,6 +10,8 @@ exclude_paths: - molecule/ - tests/azure/ - meta/runtime.yml + - requirements-docker.yml + - requirements-podman.yml kinds: - playbook: '**/tests/**/test_*.yml' diff --git a/requirements-docker.yml b/requirements-docker.yml new file mode 100644 index 00000000..660f7758 --- /dev/null +++ b/requirements-docker.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: community.docker diff --git a/requirements-podman.yml b/requirements-podman.yml new file mode 100644 index 00000000..25a97a4d --- /dev/null +++ b/requirements-podman.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: containers.podman diff --git a/tests/README.md b/tests/README.md index 65ba9762..a08068bf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -138,6 +138,35 @@ molecule destroy -s c8s See [Running the tests](#running-the-tests) section for more information on available options. +## Running local tests with upstream CI images + +To run tests locally using the same images used by upstream CI use `utils/run-tests.sh`. + +``` +utils/run-tests.sh tests/config/test_config.yml +``` + +To run all tests for a single plugin, use the `-s` option with plugin directory name. This will search, recursively for playbooks named with the pattern `test_*.yml`. To run all playbook tests for `ipauser`: + +``` +utils/run-tests.sh -s user +``` + +When executed, `utils/run-tests.sh` will create a container (either using `docker` or `podman`) using one of the testing ansible-freeipa images (https://quay.io/repository/ansible-freeipa/upstream-tests?tab=tags), run the selected tests against the container, and remove the container after tests are executed. If a test fails the container is not removed, so the failure can be investigated. + +It is possible to keep the container after the execution, even if tests succeed, using the `-C` option: + +``` +utils/run-tests.sh -s config -C +``` + +By default the tests are executed against the latest version of the Fedora image (`fedora-latest`). The testing image can be selected with the `-i` option. Use `-l` to list the available image names. + +``` +utils/run-tests.sh -i c9s tests/host/test_host.yml +``` + + ## Upcoming/desired improvements: * A script to pre-config the complete test environment using virsh. diff --git a/tests/utils.py b/tests/utils.py index eaeff3dd..eb64bef5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,10 +32,11 @@ from unittest import TestCase SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -def is_docker_env(): - if os.getenv("RUN_TESTS_IN_DOCKER", "0") == "0": - return False - return True +def get_docker_env(): + docker_env = os.getenv("RUN_TESTS_IN_DOCKER", None) + if docker_env in ["1", "True", "true", "yes", True]: + docker_env = "docker" + return docker_env def get_ssh_password(): @@ -88,8 +89,9 @@ def get_inventory_content(): """Create the content of an inventory file for a test run.""" ipa_server_host = get_server_host() - if is_docker_env(): - ipa_server_host += " ansible_connection=docker" + container_engine = get_docker_env() + if container_engine is not None: + ipa_server_host += f" ansible_connection={container_engine}" sshpass = get_ssh_password() if sshpass: @@ -145,12 +147,11 @@ def _run_playbook(playbook): with tempfile.NamedTemporaryFile() as inventory_file: inventory_file.write(get_inventory_content()) inventory_file.flush() - cmd = [ - "ansible-playbook", - "-i", - inventory_file.name, - playbook, - ] + cmd_options = ["-i", inventory_file.name] + verbose = os.environ.get("IPA_VERBOSITY", None) + if verbose is not None: + cmd_options.append(verbose) + cmd = ["ansible-playbook"] + cmd_options + [playbook] # pylint: disable=subprocess-run-check process = subprocess.run( cmd, cwd=SCRIPT_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -257,8 +258,9 @@ def kdestroy(host): class AnsibleFreeIPATestCase(TestCase): def setUp(self): - if is_docker_env(): - protocol = "docker://" + container_engine = get_docker_env() + if container_engine: + protocol = f"{container_engine}://" user = "" ssh_identity_file = None else: diff --git a/utils/run-tests.sh b/utils/run-tests.sh new file mode 100755 index 00000000..d227b3bb --- /dev/null +++ b/utils/run-tests.sh @@ -0,0 +1,400 @@ +#!/bin/bash -eu + +trap interrupt_exception SIGINT + +RST="\033[0m" +RED="\033[31m" +# BRIGHTRED="\033[31;1m" +# GREEN="\033[32m" +BRIGHTGREEN="\033[32;1m" +# BROWN="\033[33m" +YELLOW="\033[33;1m" +# NAVY="\033[34m" +BLUE="\033[34;1m" +# MAGENTA="\033[35m" +# BRIGHTMAGENTA="\033[35;1m" +# DARKCYAN="\033[36m" +# CYAN="\033[36;1m" +# BLACK="\033[30m" +# DARKGRAY="\033[30;1m" +# GRAY="\033[37m" +WHITE="\033[37;1m" + +TOPDIR="$(readlink -f "$(dirname "$0")/..")" + +interrupt_exception() { + trap - SIGINT + log warn "User interrupted test execution." + cleanup + exit 1 +} + +usage() { + local prog="${0##*/}" + cat </dev/null 2>&1 +} + +in_python_virtualenv() { + local script + read -r -d "" script <"${playbook}" + ansible-playbook -i "${inventory}" "${playbook}" + err=$? + rm "${playbook}" + return ${err} +} + +die() { + usg="N" + if [ "${1}" == "-u" ] + then + usg="Y" + shift 1 + fi + log error "${*}" + STOP_CONTAINER="N" + cleanup + [ "${usg}" == "Y" ] && usage + exit 1 +} + +make_inventory() { + local scenario=$1 engine=${2:-podman} + inventory="${test_env}/inventory" + log info "Inventory file: ${inventory}" + cat << EOF > "${inventory}" +[ipaserver] +${scenario} ansible_connection=${engine} +[ipaserver:vars] +ipaserver_domain = test.local +ipaserver_realm = TEST.LOCAL +EOF +} + +stop_container() { + local scenario=${1} engine=${2:-podman} + echo "Stopping container..." + quiet "${engine}" stop "${scenario}" + echo "Removing container..." + quiet "${engine}" rm "${scenario}" +} + +cleanup() { + if [ $# -gt 0 ] + then + if [ "${STOP_CONTAINER}" != "N" ] + then + stop_container "${1}" "${2}" + rm "${inventory}" + else + log info "Keeping container: $(podman ps --format "{{.Names}} - {{.ID}}" --filter "name=${1}")" + fi + fi + if [ "${STOP_VIRTUALENV}" == "Y" ] + then + echo "Deactivating virtual environment" + deactivate + fi +} + +list_images() { + local quay_api="https://quay.io/api/v1/repository/ansible-freeipa/upstream-tests/tag" + echo -e "${WHITE}Available images:" + curl --silent -L "${quay_api}" | jq '.tags[]|.name' | tr -d '"'| sort | uniq | sed "s/.*/ &/" + echo -e "${RST}" +} + +# Defaults + +ANSIBLE_VERSION=${ANSIBLE_VERSION:-'ansible-core>=2.12,<2.13'} +verbose="" +FORCE_ENV="N" +CONTINUE_ON_ERROR="" +STOP_CONTAINER="Y" +STOP_VIRTUALENV="N" +declare -a ENABLED_MODULES +declare -a ENABLED_TESTS +ENABLED_MODULES=() +ENABLED_TESTS=() +test_env="${TESTENV_DIR:-${VIRTUAL_ENV:-/tmp/ansible-freeipa-tests}}" +engine="podman" +IMAGE_REPO="quay.io/ansible-freeipa/upstream-tests" +IMAGE_TAG="fedora-latest" +scenario="" +MEMORY=3 +hostname="ipaserver.test.local" + +ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"containers.podman"} + +# Process command options + +while getopts ":hc:ei:Klms:v" option +do + case "$option" in + h) help && exit 0 ;; + c) scenario="${OPTARG}" ;; + e) FORCE_ENV="Y" ;; + i) IMAGE_TAG="${OPTARG}" ;; + K) STOP_CONTAINER="N" ;; + l) list_images && exit 0 || exit 1;; + m) MEMORY="${OPTARG}" ;; + s) + if [ -d "${TOPDIR}/tests/${OPTARG}" ] + then + ENABLED_MODULES+=("${OPTARG}") + else + log error "Invalid suite: ${OPTARG}" + fi + ;; + v) verbose=${verbose:--}${option} ;; + *) die -u "Invalid option: ${OPTARG}" ;; + esac +done + +for test in "${@:${OPTIND}}" +do + # shellcheck disable=SC2207 + if stat "$test" >/dev/null 2>&1 + then + ENABLED_TESTS+=($(basename "${test}" .yml)) + else + log error "Test not found: ${test}" + fi +done + +[ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined." + +# Prepare virtual environment +VENV=$(in_python_virtualenv && echo Y || echo N) + +if [ "${FORCE_ENV}" == "Y" ] +then + [ "${VENV}" == "Y" ] && deactivate + VENV="N" + rm -rf "$test_env" + log info "Virtual environment will be (re)created." +fi + +if [ "$VENV" == "N" ] +then + log info "Preparing virtual environment: ${test_env}" + if [ ! -d "${test_env}" ] + then + log info "Creating virtual environment: ${test_env}..." + if ! python3 -m venv "${test_env}" + then + die "Cannot create virtual environment." + fi + fi + if [ -f "${test_env}/bin/activate" ] + then + log info "Starting virtual environment: ${test_env}" + # shellcheck disable=SC1091 + . "${test_env}/bin/activate" || die "Cannot activate environment." + STOP_VIRTUALENV="Y" + else + die "Cannot activate environment." + fi + log info "Installing required tools." + log none "Upgrading: pip setuptools wheel" + pip install --quiet --upgrade pip setuptools wheel + log info "Installing dependencies from 'requirements-tests.txt'" + pip install --quiet -r "${TOPDIR}/requirements-tests.txt" + log info "Installing Ansible: ${ANSIBLE_VERSION}" + pip install --quiet "${ANSIBLE_VERSION}" + log debug "Ansible version: $(ansible --version | sed -n "1p")${RST}" + if [ -n "${ANSIBLE_COLLECTIONS}" ] + then + log warn "Installed collections will not be removed after execution." + log none "Installing: Ansible Collection ${ANSIBLE_COLLECTIONS}" + # shellcheck disable=SC2086 + quiet ansible-galaxy collection install ${ANSIBLE_COLLECTIONS} || die "Failed to install Ansible collections." + fi +else + log info "Using current virtual environment." +fi + +# Ansible configuration +export ANSIBLE_ROLES_PATH="${TOPDIR}/roles" +export ANSIBLE_LIBRARY="${TOPDIR}/plugins:${TOPDIR}/molecule" +export ANSIBLE_MODULE_UTILS="${TOPDIR}/plugins/module_utils" + +# Prepare container +container_id="" +container_status=("-f" "status=created" "-f" "status=running") +[ -n "${scenario}" ] && container_id="$(${engine} ps --all -q -f "name=${scenario}" "${container_status[@]}")" +if [ -z "${container_id}" ] +then + # Retrieve image and start container. + log info "Pulling FreeIPA image '${IMAGE_REPO}:${IMAGE_TAG}'..." + img_id=$(${engine} pull -q "${IMAGE_REPO}:${IMAGE_TAG}") + log info "Creating container..." + CONFIG="--hostname ${hostname} --memory ${MEMORY}g --memory-swap -1 --dns none --add-host ipaserver.test.local:127.0.0.1" + [ -n "${scenario}" ] && CONFIG="${CONFIG} --name ${scenario}" + # shellcheck disable=SC2086 + container_id=$(${engine} create ${CONFIG} "${img_id}" || die "Cannot create container") + echo "CONTAINER: ${container_id}" +fi +scenario="${scenario:-$(${engine} ps -q --format "{{.Names}}" --filter "id=${container_id}" "${container_status[@]}")}" +log debug "Using container: ${scenario}" + +# Start container +make_inventory "${scenario}" +log info "Starting container for ${scenario}..." +quiet ${engine} start "${scenario}" + +# create /etc/resolve.conf +run_inline_playbook <