From 8153239ef7eae7e0fc2dea49f13e32d243905264 Mon Sep 17 00:00:00 2001 From: Thomas Woerner Date: Fri, 19 Jul 2024 18:44:20 +0200 Subject: [PATCH 1/8] New image builder without molecule using podman The new image builder is not using molecule and uses podman directly for the generation of the ansible-test images. Two additional services are installed to simplify the use of the container in the test: - fixnet.service uses /root/fixnet.sh to fix IP address of the server in /etc/hosts and to set localhost as the nameserver. This service is executed before IPA is started. This eliminates the need to restart the IPA server after the container has been started and the IPs have been fixed. - fixipaip.service uses /root/fixipaip.sh to fix the IP address of the IPA dnsrecords of server and ipa-ca. With these services it is now only needed to wait till all services in the container are started. There is no need to restart the IPA server anymore. Simply use something like this before starting the tests: while [ -n "$(podman exec ansible-test systemctl list-jobs | grep -vi 'no jobs running')" ]; do echo "waiting.."; sleep 5; done New files - infra/image/build.sh - infra/image/dockerfile/c8s - infra/image/dockerfile/c9s - infra/image/dockerfile/c10s - infra/image/dockerfile/fedora-latest - infra/image/dockerfile/fedora-rawhide - infra/image/inventory - infra/image/system-service/fixipaip.service - infra/image/system-service/fixipaip.sh - infra/image/system-service/fixnet.service - infra/image/system-service/fixnet.sh --- infra/image/build.sh | 66 +++++++++++++++++++++ infra/image/dockerfile/c10s | 26 ++++++++ infra/image/dockerfile/c8s | 32 ++++++++++ infra/image/dockerfile/c9s | 26 ++++++++ infra/image/dockerfile/fedora-latest | 28 +++++++++ infra/image/dockerfile/fedora-rawhide | 29 +++++++++ infra/image/inventory | 15 +++++ infra/image/system-service/fixipaip.service | 10 ++++ infra/image/system-service/fixipaip.sh | 26 ++++++++ infra/image/system-service/fixnet.service | 12 ++++ infra/image/system-service/fixnet.sh | 24 ++++++++ 11 files changed, 294 insertions(+) create mode 100755 infra/image/build.sh create mode 100644 infra/image/dockerfile/c10s create mode 100644 infra/image/dockerfile/c8s create mode 100644 infra/image/dockerfile/c9s create mode 100644 infra/image/dockerfile/fedora-latest create mode 100644 infra/image/dockerfile/fedora-rawhide create mode 100644 infra/image/inventory create mode 100644 infra/image/system-service/fixipaip.service create mode 100755 infra/image/system-service/fixipaip.sh create mode 100644 infra/image/system-service/fixnet.service create mode 100755 infra/image/system-service/fixnet.sh diff --git a/infra/image/build.sh b/infra/image/build.sh new file mode 100755 index 00000000..1e31cbb0 --- /dev/null +++ b/infra/image/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash -eu + +BASEDIR="$(readlink -f "$(dirname "$0")")" +TOPDIR="$(readlink -f "${BASEDIR}/../..")" + +scenario=${1:-} +name="ansible-test" +hostname="ipaserver.test.local" +cpus="2" +memory="4g" +quayname="quay.io/ansible-freeipa/upstream-tests" + +if [ -z "${scenario}" ]; then + echo "ERROR: Image needs to be given" + exit 1 +fi +if [ ! -f "${BASEDIR}/dockerfile/${scenario}" ]; then + echo "ERROR: ${scenario} is not a valid image" + exit 1 +fi + +echo "= Cleanup existing ${scenario} =" +podman image rm "${scenario}" --force +echo + +echo "= Building ${scenario} =" +podman build -t "${scenario}" -f "${BASEDIR}/dockerfile/${scenario}" \ + "${BASEDIR}" +echo + +echo "= Creating ${name} =" +podman create --privileged --name "${name}" --hostname "${hostname}" \ + --network bridge:interface_name=eth0 --systemd true \ + --cpus "${cpus}" --memory "${memory}" --memory-swap -1 --no-hosts \ + --replace "${scenario}" +echo + +echo "= Starting ${name} =" +podman start "${name}" +echo + +echo "= Installing IPA =" +ansible-playbook -i "${BASEDIR}/inventory" \ + "${TOPDIR}/playbooks/install-server.yml" +echo + +echo "= Enabling additional services =" +podman exec "${name}" systemctl enable fixnet +podman exec "${name}" systemctl enable fixipaip +echo + +echo "= Stopping ${name} =" +podman stop "${name}" +echo + +echo "= Committing \"${quayname}:${scenario}\" =" +podman commit "${name}" "${quayname}:${scenario}" +echo + +echo "= DONE =" + +# For tests: +# podman start "${name}" +# while [ -n "$(podman exec ansible-test systemctl list-jobs | grep -vi "no jobs running")" ]; do echo "waiting.."; sleep 5; done +# # Run tests +# podman stop "${name}" diff --git a/infra/image/dockerfile/c10s b/infra/image/dockerfile/c10s new file mode 100644 index 00000000..622098f3 --- /dev/null +++ b/infra/image/dockerfile/c10s @@ -0,0 +1,26 @@ +FROM quay.io/centos/centos:stream10-development +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/dockerfile/c8s b/infra/image/dockerfile/c8s new file mode 100644 index 00000000..87a5b82e --- /dev/null +++ b/infra/image/dockerfile/c8s @@ -0,0 +1,32 @@ +FROM quay.io/centos/centos:stream8 +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo; \ +sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo; \ +sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/python3-config \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +dnf clean all; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] + diff --git a/infra/image/dockerfile/c9s b/infra/image/dockerfile/c9s new file mode 100644 index 00000000..5fe77d92 --- /dev/null +++ b/infra/image/dockerfile/c9s @@ -0,0 +1,26 @@ +FROM quay.io/centos/centos:stream9 +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/dockerfile/fedora-latest b/infra/image/dockerfile/fedora-latest new file mode 100644 index 00000000..aadcffb7 --- /dev/null +++ b/infra/image/dockerfile/fedora-latest @@ -0,0 +1,28 @@ +FROM fedora:latest +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/python3-config \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +dnf clean all; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/dockerfile/fedora-rawhide b/infra/image/dockerfile/fedora-rawhide new file mode 100644 index 00000000..5a1aa005 --- /dev/null +++ b/infra/image/dockerfile/fedora-rawhide @@ -0,0 +1,29 @@ +FROM fedora:rawhide +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/python3-config \ + /usr/bin/dnf-3 \ + python3-libdnf5 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +dnf clean all; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/inventory b/infra/image/inventory new file mode 100644 index 00000000..7ee5e709 --- /dev/null +++ b/infra/image/inventory @@ -0,0 +1,15 @@ +[ipaserver] +ansible-test ansible_connection=podman ansible_python_interpreter=/usr/bin/python3 + +[ipaserver:vars] +ipaadmin_password=SomeADMINpassword +ipadm_password=SomeDMpassword +ipaserver_domain=test.local +ipaserver_realm=TEST.LOCAL +ipaserver_setup_dns=true +ipaserver_auto_forwarders=true +ipaserver_no_dnssec_validation=true +ipaserver_auto_reverse=true +ipaserver_setup_kra=true +ipaserver_setup_firewalld=false +ipaclient_no_ntp=true diff --git a/infra/image/system-service/fixipaip.service b/infra/image/system-service/fixipaip.service new file mode 100644 index 00000000..6dde6ae8 --- /dev/null +++ b/infra/image/system-service/fixipaip.service @@ -0,0 +1,10 @@ +[Unit] +Description=Fix IPA server IP in IPA Server +After=multi-user.target + +[Service] +Type=oneshot +ExecStart=/root/fixipaip.sh + +[Install] +WantedBy=default.target diff --git a/infra/image/system-service/fixipaip.sh b/infra/image/system-service/fixipaip.sh new file mode 100755 index 00000000..dd638fa0 --- /dev/null +++ b/infra/image/system-service/fixipaip.sh @@ -0,0 +1,26 @@ +#!/bin/bash -eu + +HOSTNAME=$(hostname) +IP=$(hostname -I | cut -d " " -f 1) + +if [ -z "${HOSTNAME}" ]; then + echo "ERROR: Failed to retrieve hostname." + exit 1 +fi +if [ -z "${IP}" ]; then + echo "ERROR: Failed to retrieve IP address." + exit 1 +fi + +if ! echo "SomeADMINpassword" | kinit -c ansible_freeipa_cache admin +then + echo "ERROR: Failed to obtain Kerberos ticket" + exit 1 +fi +KRB5CCNAME=ansible_freeipa_cache \ + ipa dnsrecord-mod test.local "${HOSTNAME%%.*}" --a-rec="$IP" +KRB5CCNAME=ansible_freeipa_cache \ + ipa dnsrecord-mod test.local ipa-ca --a-rec="$IP" +kdestroy -c ansible_freeipa_cache -A + +exit 0 diff --git a/infra/image/system-service/fixnet.service b/infra/image/system-service/fixnet.service new file mode 100644 index 00000000..c481b19e --- /dev/null +++ b/infra/image/system-service/fixnet.service @@ -0,0 +1,12 @@ +[Unit] +Description=Fix server IP in IPA Server +Wants=network.target +After=network.target +Before=ipa.service + +[Service] +Type=oneshot +ExecStart=/root/fixnet.sh + +[Install] +WantedBy=ipa.service diff --git a/infra/image/system-service/fixnet.sh b/infra/image/system-service/fixnet.sh new file mode 100755 index 00000000..3fc05b51 --- /dev/null +++ b/infra/image/system-service/fixnet.sh @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +HOSTNAME=$(hostname) +IP=$(hostname -I | cut -d " " -f 1) + +if [ -z "${HOSTNAME}" ]; then + echo "ERROR: Failed to retrieve hostname." + exit 1 +fi +if [ -z "${IP}" ]; then + echo "ERROR: Failed to retrieve IP address." + exit 1 +fi + +# shellcheck disable=SC2143 +if [ -n "$(grep -P "[[:space:]]${HOSTNAME}" /etc/hosts)" ]; then + sed -ie "s/.*${HOSTNAME}/${IP}\t${HOSTNAME}/" /etc/hosts +else + echo -e "$IP\t${HOSTNAME}" >> /etc/hosts +fi + +echo "nameserver 127.0.0.1" > /etc/resolv.conf + +exit 0 From 7ac0ec6bd47ad553add9053e989430646b48b344 Mon Sep 17 00:00:00 2001 From: Thomas Woerner Date: Sat, 20 Jul 2024 22:15:25 +0200 Subject: [PATCH 2/8] tests/azure/templates/build_container.yml: Use new image builder The new infra/image/build.sh script is used instead of molecule. --- tests/azure/templates/build_container.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/azure/templates/build_container.yml b/tests/azure/templates/build_container.yml index 2b47c1b4..18c1bd8a 100644 --- a/tests/azure/templates/build_container.yml +++ b/tests/azure/templates/build_container.yml @@ -22,21 +22,17 @@ jobs: retryCountOnTaskFailure: 5 displayName: Install tools - - script: pip install molecule-plugins[docker] "requests<2.29" + - script: ansible-galaxy collection install containers.podman retryCountOnTaskFailure: 5 - displayName: Install molecule + displayName: Install Ansible collections - - script: molecule create -s ${{ parameters.build_scenario_name }} + - script: infra/image/build.sh ${{ parameters.build_scenario_name }} retryCountOnTaskFailure: 5 displayName: Create test container - env: - ANSIBLE_LIBRARY: ./molecule - script: | - docker stop ${{ parameters.build_scenario_name }} - docker commit ${{ parameters.build_scenario_name }} quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }} - docker login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io - docker push quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }} + podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io + podman push quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }} displayName: Save image and upload env: # Secrets needs to be mapped as env vars to work properly From b0e03a032df9d90283294e6d2b6319c1c1864dd8 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Fri, 19 Jul 2024 17:45:47 -0300 Subject: [PATCH 3/8] Add shell utitily functions for scripts Add two shell libaries with utilities to write shell scripts. The 'utils/shlog' file provides macro names for ANSI escape sequences to control color output on terminals, a 'log' functions with pre-defined behavior for ERROR, WARN, DEBUG, INFO and SUCCESS level messages, and the 'quiet' function which executes a command and hides its output. The 'utils/shfun' file provides an interruptinon handler for SIGINT, and the following functions: - run_if_exists: run a command if it is available - cleanup: cleanup environment, possibly stopping a container and a Python virtual environment. - start_virtual_environmnt: initiates a Python virtual environment - in_python_virtualenv: test if the script is running inside a Python virtual environment - die: abort the script with an error message end exit code 1 New files: - utils/shlog - utils/shfun --- utils/shfun | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++ utils/shlog | 57 +++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 utils/shfun create mode 100644 utils/shlog diff --git a/utils/shfun b/utils/shfun new file mode 100644 index 00000000..e14b9bd6 --- /dev/null +++ b/utils/shfun @@ -0,0 +1,118 @@ +#!/bin/bash -eu +# This file is meant to be source'd by shell scripts + +SCRIPTDIR="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")" + +. "${SCRIPTDIR}/shlog" + +[ -n "$(command -v python3)" ] && python="$(command -v python3)" || python="$(command -v python2)" +export python + +trap interrupt_exception SIGINT + +interrupt_exception() { + trap - SIGINT + log warn "User interrupted test execution." + # shellcheck disable=SC2119 + cleanup "${scenario:+${scenario}}" + exit 1 +} + +run_if_exists() { + cmd="${1}" + shift + [ -n "$(command -v "${cmd}")" ] && "${cmd}" "${@}" +} + +# shellcheck disable=SC2120 +cleanup() { + local container container_engine + container="${1:-${scenario:+${scenario}}}" + container_engine="${2:-${engine:-"podman"}}" + if [ "${STOP_CONTAINER:-"Y"}" == "Y" ] && [ -n "${container}" ] + then + run_if_exists stop_container "${container}" "${container_engine}" + [ -f "${inventory:-}" ] && rm "${inventory}" + else + if [ -n "${container}" ] + then + log info "Keeping container: $(${container_engine} ps --format "{{.Names}} - {{.ID}}" --filter "name=${container}")" + fi + fi + if [ "${STOP_VIRTUALENV:-"N"}" == "Y" ] + then + echo "Deactivating virtual environment" + run_if_exists deactivate + fi +} + +start_virtual_environment() { + # options -f + local FORCE_ENV VENV envdirectory + FORCE_ENV="N" + while getopts ":f" option + do + case "$option" in + f) FORCE_ENV="Y" ;; + *) die "prepare_virtual_environment: Invalid option: ${option}" ;; + esac + done + envdirectory="${test_env:-/tmp/ansible-freeipa-tests}" + + # Prepare virtual environment + VENV=$(in_python_virtualenv && echo Y || echo N) + + if [ "${FORCE_ENV}" == "Y" ] + then + run_if_exists deactivate + VENV="N" + rm -rf "$test_env" + log info "Virtual environment will be (re)created." + fi + + if [ "$VENV" == "N" ] + then + log info "Preparing virtual environment: ${envdirectory}" + if [ ! -d "${envdirectory}" ] || [ ! -f "${envdirectory}/bin/activate" ] + then + log info "Creating virtual environment: ${envdirectory}..." + log warn "RUN: ${python} -m venv ${envdirectory}" + ${python} -m venv "${envdirectory}" || die "Cannot create virtual environment." + fi + log info "Starting virtual environment: ${envdirectory}" + [ -f "${envdirectory}/bin/activate" ] || die "Failed to create virtual environment." + # shellcheck disable=SC1091 + . "${envdirectory}/bin/activate" || die "Cannot activate virtual environment." + export STOP_VIRTUALENV="Y" + log info "Installing required tools." + log none "Upgrading: pip setuptools wheel" + pip install --quiet --upgrade pip setuptools wheel + else + log info "Using current virtual environment." + fi +} + +die() { + usg="N" + if [ "${1}" == "-u" ] + then + usg="Y" + shift 1 + fi + log error "${*}" + STOP_CONTAINER="N" + cleanup "${scenario:+${scenario}}" + [ "${usg}" == "Y" ] && run_if_exists usage + exit 1 +} + +in_python_virtualenv() { + local script + read -r -d "" script </dev/null 2>&1 +} + From fb6fed58cb3f704dffbe6a3d06a6f3d4cc851670 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Fri, 19 Jul 2024 17:57:30 -0300 Subject: [PATCH 4/8] rjeffman: this is a fixup for infra/images/build.sh This patch modifies the image building script by adding: - An usage message. - An option "-I" to NOT install IPA to the generated container. - An opiton "-c NAME" to both set the name and use an existing container to ONLY install IPA. - Rename "scenario" to "DISTRO" as "scenario" should be used for the container scenario usage, rather than the distro (I'll change the name also in the Azure scripts) - Use 'log' (from shlog) to print messages. --- infra/image/build.sh | 146 +++++++++++++++++++++++++++++++----------- infra/image/inventory | 2 +- 2 files changed, 111 insertions(+), 37 deletions(-) diff --git a/infra/image/build.sh b/infra/image/build.sh index 1e31cbb0..4f77b848 100755 --- a/infra/image/build.sh +++ b/infra/image/build.sh @@ -3,61 +3,135 @@ BASEDIR="$(readlink -f "$(dirname "$0")")" TOPDIR="$(readlink -f "${BASEDIR}/../..")" -scenario=${1:-} -name="ansible-test" +. "${TOPDIR}/utils/shfun" + +valid_distro() { + find "${BASEDIR}/dockerfile" -type f -printf "%f\n" | tr "\n" " " +} + +usage() { + local prog="${0##*/}" + cat << EOF +usage: ${prog} [-h] [i] distro + ${prog} build a container image to test ansible-freeipa. +EOF +} + +help() { + cat << EOF +positional arguments: + + distro The base distro to build the test container. + Availble distros: $(valid_distro) + +optional arguments: + + -s Deploy IPA server + +EOF +} + +name="ansible-freeipa-image-builder" hostname="ipaserver.test.local" -cpus="2" +# Number of cpus is not available in usptream CI (Ubuntu 22.04). +# cpus="2" memory="4g" quayname="quay.io/ansible-freeipa/upstream-tests" +deploy_server="N" -if [ -z "${scenario}" ]; then - echo "ERROR: Image needs to be given" - exit 1 -fi -if [ ! -f "${BASEDIR}/dockerfile/${scenario}" ]; then - echo "ERROR: ${scenario} is not a valid image" - exit 1 +while getopts ":hs" option +do + case "${option}" in + h) help && exit 0 ;; + s) deploy_server="Y" ;; + *) die -u "Invalid option: ${option}" ;; + esac +done + +shift $((OPTIND - 1)) +distro=${1:-} + +[ -n "${distro}" ] || die "Distro needs to be given.\nUse one of: $(valid_distro)" + +[ -f "${BASEDIR}/dockerfile/${distro}" ] \ + || die "${distro} is not a valid distro target.\nUse one of: $(valid_distro)" + +if [ "${deploy_server}" == "Y" ] +then + [ -n "$(command -v "ansible-playbook")" ] || die "ansible-playbook is required to install FreeIPA." + + deploy_playbook="${TOPDIR}/playbooks/install-server.yml" + [ -f "${deploy_playbook}" ] || die "Can't find playbook '${deploy_playbook}'" + + inventory_file="${BASEDIR}/inventory" + [ -f "${inventory_file}" ] || die "Can't find inventory '${inventory_file}'" fi -echo "= Cleanup existing ${scenario} =" -podman image rm "${scenario}" --force -echo +container_state="$(podman ps -q --all --format "{{.State}}" --filter "name=${name}")" -echo "= Building ${scenario} =" -podman build -t "${scenario}" -f "${BASEDIR}/dockerfile/${scenario}" \ +tag="${distro}-base" +server_tag="${distro}-server" + +# in older (as in Ubuntu 22.04) podman versions, +# 'podman image rm --force' fails if the image +# does not exist. +remove_image_if_exists() +{ + local tag_to_remove + tag_to_remove="${1}" + if podman image exists "${tag_to_remove}" + then + log info "= Cleanup ${tag_to_remove} =" + podman image rm "${tag_to_remove}" --force + echo + fi +} + +remove_image_if_exists "${tag}" +[ "${deploy_server}" == "Y" ] && remove_image_if_exists "${server_tag}" + + +log info "= Building ${tag} =" +podman build -t "${tag}" -f "${BASEDIR}/dockerfile/${distro}" \ "${BASEDIR}" echo -echo "= Creating ${name} =" +log info "= Creating ${name} =" podman create --privileged --name "${name}" --hostname "${hostname}" \ - --network bridge:interface_name=eth0 --systemd true \ - --cpus "${cpus}" --memory "${memory}" --memory-swap -1 --no-hosts \ - --replace "${scenario}" + --network bridge:interface_name=eth0 --systemd true \ + --memory "${memory}" --memory-swap -1 --no-hosts \ + --replace "${tag}" echo -echo "= Starting ${name} =" -podman start "${name}" +log info "= Committing \"${quayname}:${tag}\" =" +podman commit "${name}" "${quayname}:${tag}" echo -echo "= Installing IPA =" -ansible-playbook -i "${BASEDIR}/inventory" \ - "${TOPDIR}/playbooks/install-server.yml" -echo +if [ "${deploy_server}" == "Y" ] +then + log info "= Starting ${name} =" + [ "${container_state}" == "running" ] || podman start "${name}" + echo -echo "= Enabling additional services =" -podman exec "${name}" systemctl enable fixnet -podman exec "${name}" systemctl enable fixipaip -echo + log info "= Deploying IPA =" + ansible-playbook -i "${inventory_file}" "${deploy_playbook}" + echo -echo "= Stopping ${name} =" -podman stop "${name}" -echo + log info "= Enabling additional services =" + podman exec "${name}" systemctl enable fixnet + podman exec "${name}" systemctl enable fixipaip + echo + + log info "= Stopping container ${name} =" + podman stop "${name}" + echo -echo "= Committing \"${quayname}:${scenario}\" =" -podman commit "${name}" "${quayname}:${scenario}" -echo + log info "= Committing \"${quayname}:${server_tag}\" =" + podman commit "${name}" "${quayname}:${server_tag}" + echo +fi -echo "= DONE =" +log info "= DONE: Image created. =" # For tests: # podman start "${name}" diff --git a/infra/image/inventory b/infra/image/inventory index 7ee5e709..4a83eb75 100644 --- a/infra/image/inventory +++ b/infra/image/inventory @@ -1,5 +1,5 @@ [ipaserver] -ansible-test ansible_connection=podman ansible_python_interpreter=/usr/bin/python3 +ansible-freeipa-image-builder ansible_connection=podman ansible_python_interpreter=/usr/bin/python3 [ipaserver:vars] ipaadmin_password=SomeADMINpassword From 928ed30b8b744e910d03af7ae1a97915e76b8094 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Fri, 19 Jul 2024 22:56:21 -0300 Subject: [PATCH 5/8] ustream ci: Use infra scripts to build testing images As we do not use molecule features, using a Dockerfile and the ansible-freeipa deployment roles is enough to create the container testing images. This patch removes the usage of molecule in favor of the custom ansible-freeipa image building script, which allow us to have a similar process for creating images both on the ustream CI, or on a developer's environment. Also, CentOS 7 is removed from the build script, as it in not possible to run CentOS 7 containers with current versions of systemd. --- tests/azure/build-containers.yml | 33 +++++++++-------- tests/azure/templates/build_container.yml | 43 ++++++++++++++--------- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/tests/azure/build-containers.yml b/tests/azure/build-containers.yml index 0423dfd9..75c38539 100644 --- a/tests/azure/build-containers.yml +++ b/tests/azure/build-containers.yml @@ -11,18 +11,19 @@ schedules: trigger: none pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' stages: -- stage: CentOS_7 - dependsOn: [] - jobs: - - template: templates/build_container.yml - parameters: - job_name_suffix: Centos7 - container_name: centos-7 - build_scenario_name: centos-7-build +# Currently, it's not possible to use CentOS container +# +# - stage: CentOS_7 +# dependsOn: [] +# jobs: +# - template: templates/build_container.yml +# parameters: +# job_name_suffix: Centos7 +# distro: centos-7 - stage: CentOS_8_Stream dependsOn: [] @@ -30,8 +31,9 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: C8S - container_name: c8s - build_scenario_name: c8s-build + distro: c8s + # ansible-core 2.17+ cannot be used to deploy on CentOS 8 Stream. + ansible_core_version: "<2.17" - stage: CentOS_9_Stream dependsOn: [] @@ -39,8 +41,7 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: C9S - container_name: c9s - build_scenario_name: c9s-build + distro: c9s - stage: Fedora_Latest dependsOn: [] @@ -48,8 +49,7 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: FedoraLatest - container_name: fedora-latest - build_scenario_name: fedora-latest-build + distro: fedora-latest - stage: Fedora_Rawhide dependsOn: [] @@ -57,5 +57,4 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: FedoraRawhide - container_name: fedora-rawhide - build_scenario_name: fedora-rawhide-build + distro: fedora-rawhide diff --git a/tests/azure/templates/build_container.yml b/tests/azure/templates/build_container.yml index 18c1bd8a..010b4962 100644 --- a/tests/azure/templates/build_container.yml +++ b/tests/azure/templates/build_container.yml @@ -2,38 +2,49 @@ parameters: - name: job_name_suffix type: string - - name: container_name - type: string - - name: build_scenario_name + - name: distro type: string - name: python_version type: string default: 3.x + - name: ansible_core_version + default: "" jobs: - job: BuildTestImage${{ parameters.job_name_suffix }} - displayName: Build ${{ parameters.container_name }} test container + displayName: Build ${{ parameters.distro }} test container steps: - task: UsePythonVersion@0 inputs: versionSpec: '${{ parameters.python_version }}' - - script: python -m pip install --upgrade pip setuptools wheel ansible + - script: python -m pip install --upgrade pip "ansible-core${{ parameters.ansible_core_version }}" retryCountOnTaskFailure: 5 displayName: Install tools - - script: ansible-galaxy collection install containers.podman - retryCountOnTaskFailure: 5 - displayName: Install Ansible collections - - - script: infra/image/build.sh ${{ parameters.build_scenario_name }} - retryCountOnTaskFailure: 5 - displayName: Create test container - - script: | - podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io - podman push quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }} - displayName: Save image and upload + rm -rf ~/.ansible + mkdir -p ~/.ansible + ln -snf $(readlink -f roles) ~/.ansible/roles + ln -snf $(readlink -f plugins) ~/.ansible/plugins + displayName: Setup ansible-freeipa using Git repository + + - script: ansible-galaxy collection install containers.podman + displayName: Install Ansible Galaxy collections + + - script: infra/image/build.sh -s ${{ parameters.distro }} + displayName: Build ${{ parameters.distro }} base image + + - script: podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io + displayName: Registry login env: # Secrets needs to be mapped as env vars to work properly QUAY_ROBOT_TOKEN: $(QUAY_ROBOT_TOKEN) + + - script: | + podman push ${{parameters.distro}}-base quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-base + displayName: Push base image + + - script: | + podman push ${{ parameters.distro }}-server quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server + displayName: Push server image From 480c83f50427062bcdd316b16f2d4db01b6f5132 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Mon, 22 Jul 2024 12:21:55 -0300 Subject: [PATCH 6/8] fixup! New image builder without molecule using podman --- tests/azure/templates/build_container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/azure/templates/build_container.yml b/tests/azure/templates/build_container.yml index 010b4962..f93881ce 100644 --- a/tests/azure/templates/build_container.yml +++ b/tests/azure/templates/build_container.yml @@ -42,9 +42,9 @@ jobs: QUAY_ROBOT_TOKEN: $(QUAY_ROBOT_TOKEN) - script: | - podman push ${{parameters.distro}}-base quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-base + podman push quay.io/ansible-freeipa/upstream-tests:${{parameters.distro}}-base quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-base displayName: Push base image - script: | - podman push ${{ parameters.distro }}-server quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server + podman push quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server displayName: Push server image From 785681f10082029bbf90fc225e6ad112f46d5082 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Sat, 29 Jun 2024 14:50:04 +0000 Subject: [PATCH 7/8] ci lint: Allow ShellCheck to test source-d scripts. As the scripts 'utils/run-tests.sh' and 'utils/setup_test_container.sh' use some scripts as function libraries, this change forces shellcheck to also verify those scripts. --- .github/workflows/lint.yml | 2 ++ .pre-commit-config.yaml | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cec6196c..3eb9305f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -88,3 +88,5 @@ jobs: fetch-depth: 1 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master + env: + SHELLCHECK_OPTS: -x diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82bfbcd7..23fa16ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,4 +54,7 @@ repos: name: ShellCheck language: system entry: shellcheck - files: \.sh$ + args: ['-x'] + files: > + \.sh$ + utils/sh*$ From 0d246b1c111143351fd264a0918f0d04b91008fd Mon Sep 17 00:00:00 2001 From: Thomas Woerner Date: Wed, 31 Jul 2024 15:33:19 +0200 Subject: [PATCH 8/8] infra/image/build.sh: Fail if deployment failed or podman is missing If the deployment was enabled and failed, the script still continued without failing. If podman was missing it failed without a proper error. The script now fails and does not enable the services and also does not commit after the failed deployment. Also is fails if podman is missing. --- infra/image/build.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/infra/image/build.sh b/infra/image/build.sh index 4f77b848..b1602174 100755 --- a/infra/image/build.sh +++ b/infra/image/build.sh @@ -56,6 +56,8 @@ distro=${1:-} [ -f "${BASEDIR}/dockerfile/${distro}" ] \ || die "${distro} is not a valid distro target.\nUse one of: $(valid_distro)" +[ -n "$(command -v "podman")" ] || die "podman is required." + if [ "${deploy_server}" == "Y" ] then [ -n "$(command -v "ansible-playbook")" ] || die "ansible-playbook is required to install FreeIPA." @@ -109,23 +111,32 @@ echo if [ "${deploy_server}" == "Y" ] then + deployed=false + log info "= Starting ${name} =" [ "${container_state}" == "running" ] || podman start "${name}" echo log info "= Deploying IPA =" - ansible-playbook -i "${inventory_file}" "${deploy_playbook}" + if ansible-playbook -i "${inventory_file}" "${deploy_playbook}" + then + deployed=true + fi echo - log info "= Enabling additional services =" - podman exec "${name}" systemctl enable fixnet - podman exec "${name}" systemctl enable fixipaip - echo + if $deployed; then + log info "= Enabling additional services =" + podman exec "${name}" systemctl enable fixnet + podman exec "${name}" systemctl enable fixipaip + echo + fi log info "= Stopping container ${name} =" podman stop "${name}" echo + $deployed || die "Deployment failed" + log info "= Committing \"${quayname}:${server_tag}\" =" podman commit "${name}" "${quayname}:${server_tag}" echo