diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cec6196c..3eb9305f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -88,3 +88,5 @@ jobs: fetch-depth: 1 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master + env: + SHELLCHECK_OPTS: -x diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82bfbcd7..23fa16ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,4 +54,7 @@ repos: name: ShellCheck language: system entry: shellcheck - files: \.sh$ + args: ['-x'] + files: > + \.sh$ + utils/sh*$ diff --git a/infra/image/build.sh b/infra/image/build.sh new file mode 100755 index 00000000..b1602174 --- /dev/null +++ b/infra/image/build.sh @@ -0,0 +1,151 @@ +#!/bin/bash -eu + +BASEDIR="$(readlink -f "$(dirname "$0")")" +TOPDIR="$(readlink -f "${BASEDIR}/../..")" + +. "${TOPDIR}/utils/shfun" + +valid_distro() { + find "${BASEDIR}/dockerfile" -type f -printf "%f\n" | tr "\n" " " +} + +usage() { + local prog="${0##*/}" + cat << EOF +usage: ${prog} [-h] [i] distro + ${prog} build a container image to test ansible-freeipa. +EOF +} + +help() { + cat << EOF +positional arguments: + + distro The base distro to build the test container. + Availble distros: $(valid_distro) + +optional arguments: + + -s Deploy IPA server + +EOF +} + +name="ansible-freeipa-image-builder" +hostname="ipaserver.test.local" +# Number of cpus is not available in usptream CI (Ubuntu 22.04). +# cpus="2" +memory="4g" +quayname="quay.io/ansible-freeipa/upstream-tests" +deploy_server="N" + +while getopts ":hs" option +do + case "${option}" in + h) help && exit 0 ;; + s) deploy_server="Y" ;; + *) die -u "Invalid option: ${option}" ;; + esac +done + +shift $((OPTIND - 1)) +distro=${1:-} + +[ -n "${distro}" ] || die "Distro needs to be given.\nUse one of: $(valid_distro)" + +[ -f "${BASEDIR}/dockerfile/${distro}" ] \ + || die "${distro} is not a valid distro target.\nUse one of: $(valid_distro)" + +[ -n "$(command -v "podman")" ] || die "podman is required." + +if [ "${deploy_server}" == "Y" ] +then + [ -n "$(command -v "ansible-playbook")" ] || die "ansible-playbook is required to install FreeIPA." + + deploy_playbook="${TOPDIR}/playbooks/install-server.yml" + [ -f "${deploy_playbook}" ] || die "Can't find playbook '${deploy_playbook}'" + + inventory_file="${BASEDIR}/inventory" + [ -f "${inventory_file}" ] || die "Can't find inventory '${inventory_file}'" +fi + +container_state="$(podman ps -q --all --format "{{.State}}" --filter "name=${name}")" + +tag="${distro}-base" +server_tag="${distro}-server" + +# in older (as in Ubuntu 22.04) podman versions, +# 'podman image rm --force' fails if the image +# does not exist. +remove_image_if_exists() +{ + local tag_to_remove + tag_to_remove="${1}" + if podman image exists "${tag_to_remove}" + then + log info "= Cleanup ${tag_to_remove} =" + podman image rm "${tag_to_remove}" --force + echo + fi +} + +remove_image_if_exists "${tag}" +[ "${deploy_server}" == "Y" ] && remove_image_if_exists "${server_tag}" + + +log info "= Building ${tag} =" +podman build -t "${tag}" -f "${BASEDIR}/dockerfile/${distro}" \ + "${BASEDIR}" +echo + +log info "= Creating ${name} =" +podman create --privileged --name "${name}" --hostname "${hostname}" \ + --network bridge:interface_name=eth0 --systemd true \ + --memory "${memory}" --memory-swap -1 --no-hosts \ + --replace "${tag}" +echo + +log info "= Committing \"${quayname}:${tag}\" =" +podman commit "${name}" "${quayname}:${tag}" +echo + +if [ "${deploy_server}" == "Y" ] +then + deployed=false + + log info "= Starting ${name} =" + [ "${container_state}" == "running" ] || podman start "${name}" + echo + + log info "= Deploying IPA =" + if ansible-playbook -i "${inventory_file}" "${deploy_playbook}" + then + deployed=true + fi + echo + + if $deployed; then + log info "= Enabling additional services =" + podman exec "${name}" systemctl enable fixnet + podman exec "${name}" systemctl enable fixipaip + echo + fi + + log info "= Stopping container ${name} =" + podman stop "${name}" + echo + + $deployed || die "Deployment failed" + + log info "= Committing \"${quayname}:${server_tag}\" =" + podman commit "${name}" "${quayname}:${server_tag}" + echo +fi + +log info "= DONE: Image created. =" + +# For tests: +# podman start "${name}" +# while [ -n "$(podman exec ansible-test systemctl list-jobs | grep -vi "no jobs running")" ]; do echo "waiting.."; sleep 5; done +# # Run tests +# podman stop "${name}" diff --git a/infra/image/dockerfile/c10s b/infra/image/dockerfile/c10s new file mode 100644 index 00000000..622098f3 --- /dev/null +++ b/infra/image/dockerfile/c10s @@ -0,0 +1,26 @@ +FROM quay.io/centos/centos:stream10-development +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/dockerfile/c8s b/infra/image/dockerfile/c8s new file mode 100644 index 00000000..87a5b82e --- /dev/null +++ b/infra/image/dockerfile/c8s @@ -0,0 +1,32 @@ +FROM quay.io/centos/centos:stream8 +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo; \ +sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo; \ +sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/python3-config \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +dnf clean all; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] + diff --git a/infra/image/dockerfile/c9s b/infra/image/dockerfile/c9s new file mode 100644 index 00000000..5fe77d92 --- /dev/null +++ b/infra/image/dockerfile/c9s @@ -0,0 +1,26 @@ +FROM quay.io/centos/centos:stream9 +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/dockerfile/fedora-latest b/infra/image/dockerfile/fedora-latest new file mode 100644 index 00000000..aadcffb7 --- /dev/null +++ b/infra/image/dockerfile/fedora-latest @@ -0,0 +1,28 @@ +FROM fedora:latest +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/python3-config \ + /usr/bin/dnf-3 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +dnf clean all; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/dockerfile/fedora-rawhide b/infra/image/dockerfile/fedora-rawhide new file mode 100644 index 00000000..5a1aa005 --- /dev/null +++ b/infra/image/dockerfile/fedora-rawhide @@ -0,0 +1,29 @@ +FROM fedora:rawhide +ENV container=podman + +RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ +dnf makecache; \ +dnf --assumeyes install \ + /usr/bin/python3 \ + /usr/bin/python3-config \ + /usr/bin/dnf-3 \ + python3-libdnf5 \ + sudo \ + bash \ + systemd \ + procps-ng \ + iproute; \ +dnf clean all; \ +rm -rf /var/cache/dnf/; + +COPY system-service/fixnet.sh /root/ +COPY system-service/fixipaip.sh /root/ +COPY system-service/fixnet.service /etc/systemd/system/ +COPY system-service/fixipaip.service /etc/systemd/system/ +RUN chmod +x /root/fixnet.sh /root/fixipaip.sh + +STOPSIGNAL RTMIN+3 + +VOLUME ["/sys/fs/cgroup"] + +CMD ["/usr/sbin/init"] diff --git a/infra/image/inventory b/infra/image/inventory new file mode 100644 index 00000000..4a83eb75 --- /dev/null +++ b/infra/image/inventory @@ -0,0 +1,15 @@ +[ipaserver] +ansible-freeipa-image-builder ansible_connection=podman ansible_python_interpreter=/usr/bin/python3 + +[ipaserver:vars] +ipaadmin_password=SomeADMINpassword +ipadm_password=SomeDMpassword +ipaserver_domain=test.local +ipaserver_realm=TEST.LOCAL +ipaserver_setup_dns=true +ipaserver_auto_forwarders=true +ipaserver_no_dnssec_validation=true +ipaserver_auto_reverse=true +ipaserver_setup_kra=true +ipaserver_setup_firewalld=false +ipaclient_no_ntp=true diff --git a/infra/image/system-service/fixipaip.service b/infra/image/system-service/fixipaip.service new file mode 100644 index 00000000..6dde6ae8 --- /dev/null +++ b/infra/image/system-service/fixipaip.service @@ -0,0 +1,10 @@ +[Unit] +Description=Fix IPA server IP in IPA Server +After=multi-user.target + +[Service] +Type=oneshot +ExecStart=/root/fixipaip.sh + +[Install] +WantedBy=default.target diff --git a/infra/image/system-service/fixipaip.sh b/infra/image/system-service/fixipaip.sh new file mode 100755 index 00000000..dd638fa0 --- /dev/null +++ b/infra/image/system-service/fixipaip.sh @@ -0,0 +1,26 @@ +#!/bin/bash -eu + +HOSTNAME=$(hostname) +IP=$(hostname -I | cut -d " " -f 1) + +if [ -z "${HOSTNAME}" ]; then + echo "ERROR: Failed to retrieve hostname." + exit 1 +fi +if [ -z "${IP}" ]; then + echo "ERROR: Failed to retrieve IP address." + exit 1 +fi + +if ! echo "SomeADMINpassword" | kinit -c ansible_freeipa_cache admin +then + echo "ERROR: Failed to obtain Kerberos ticket" + exit 1 +fi +KRB5CCNAME=ansible_freeipa_cache \ + ipa dnsrecord-mod test.local "${HOSTNAME%%.*}" --a-rec="$IP" +KRB5CCNAME=ansible_freeipa_cache \ + ipa dnsrecord-mod test.local ipa-ca --a-rec="$IP" +kdestroy -c ansible_freeipa_cache -A + +exit 0 diff --git a/infra/image/system-service/fixnet.service b/infra/image/system-service/fixnet.service new file mode 100644 index 00000000..c481b19e --- /dev/null +++ b/infra/image/system-service/fixnet.service @@ -0,0 +1,12 @@ +[Unit] +Description=Fix server IP in IPA Server +Wants=network.target +After=network.target +Before=ipa.service + +[Service] +Type=oneshot +ExecStart=/root/fixnet.sh + +[Install] +WantedBy=ipa.service diff --git a/infra/image/system-service/fixnet.sh b/infra/image/system-service/fixnet.sh new file mode 100755 index 00000000..3fc05b51 --- /dev/null +++ b/infra/image/system-service/fixnet.sh @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +HOSTNAME=$(hostname) +IP=$(hostname -I | cut -d " " -f 1) + +if [ -z "${HOSTNAME}" ]; then + echo "ERROR: Failed to retrieve hostname." + exit 1 +fi +if [ -z "${IP}" ]; then + echo "ERROR: Failed to retrieve IP address." + exit 1 +fi + +# shellcheck disable=SC2143 +if [ -n "$(grep -P "[[:space:]]${HOSTNAME}" /etc/hosts)" ]; then + sed -ie "s/.*${HOSTNAME}/${IP}\t${HOSTNAME}/" /etc/hosts +else + echo -e "$IP\t${HOSTNAME}" >> /etc/hosts +fi + +echo "nameserver 127.0.0.1" > /etc/resolv.conf + +exit 0 diff --git a/tests/azure/build-containers.yml b/tests/azure/build-containers.yml index 0423dfd9..75c38539 100644 --- a/tests/azure/build-containers.yml +++ b/tests/azure/build-containers.yml @@ -11,18 +11,19 @@ schedules: trigger: none pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' stages: -- stage: CentOS_7 - dependsOn: [] - jobs: - - template: templates/build_container.yml - parameters: - job_name_suffix: Centos7 - container_name: centos-7 - build_scenario_name: centos-7-build +# Currently, it's not possible to use CentOS container +# +# - stage: CentOS_7 +# dependsOn: [] +# jobs: +# - template: templates/build_container.yml +# parameters: +# job_name_suffix: Centos7 +# distro: centos-7 - stage: CentOS_8_Stream dependsOn: [] @@ -30,8 +31,9 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: C8S - container_name: c8s - build_scenario_name: c8s-build + distro: c8s + # ansible-core 2.17+ cannot be used to deploy on CentOS 8 Stream. + ansible_core_version: "<2.17" - stage: CentOS_9_Stream dependsOn: [] @@ -39,8 +41,7 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: C9S - container_name: c9s - build_scenario_name: c9s-build + distro: c9s - stage: Fedora_Latest dependsOn: [] @@ -48,8 +49,7 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: FedoraLatest - container_name: fedora-latest - build_scenario_name: fedora-latest-build + distro: fedora-latest - stage: Fedora_Rawhide dependsOn: [] @@ -57,5 +57,4 @@ stages: - template: templates/build_container.yml parameters: job_name_suffix: FedoraRawhide - container_name: fedora-rawhide - build_scenario_name: fedora-rawhide-build + distro: fedora-rawhide diff --git a/tests/azure/templates/build_container.yml b/tests/azure/templates/build_container.yml index 2b47c1b4..f93881ce 100644 --- a/tests/azure/templates/build_container.yml +++ b/tests/azure/templates/build_container.yml @@ -2,42 +2,49 @@ parameters: - name: job_name_suffix type: string - - name: container_name - type: string - - name: build_scenario_name + - name: distro type: string - name: python_version type: string default: 3.x + - name: ansible_core_version + default: "" jobs: - job: BuildTestImage${{ parameters.job_name_suffix }} - displayName: Build ${{ parameters.container_name }} test container + displayName: Build ${{ parameters.distro }} test container steps: - task: UsePythonVersion@0 inputs: versionSpec: '${{ parameters.python_version }}' - - script: python -m pip install --upgrade pip setuptools wheel ansible + - script: python -m pip install --upgrade pip "ansible-core${{ parameters.ansible_core_version }}" retryCountOnTaskFailure: 5 displayName: Install tools - - script: pip install molecule-plugins[docker] "requests<2.29" - retryCountOnTaskFailure: 5 - displayName: Install molecule - - - script: molecule create -s ${{ parameters.build_scenario_name }} - retryCountOnTaskFailure: 5 - displayName: Create test container - env: - ANSIBLE_LIBRARY: ./molecule - - script: | - docker stop ${{ parameters.build_scenario_name }} - docker commit ${{ parameters.build_scenario_name }} quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }} - docker login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io - docker push quay.io/ansible-freeipa/upstream-tests:${{ parameters.container_name }} - displayName: Save image and upload + rm -rf ~/.ansible + mkdir -p ~/.ansible + ln -snf $(readlink -f roles) ~/.ansible/roles + ln -snf $(readlink -f plugins) ~/.ansible/plugins + displayName: Setup ansible-freeipa using Git repository + + - script: ansible-galaxy collection install containers.podman + displayName: Install Ansible Galaxy collections + + - script: infra/image/build.sh -s ${{ parameters.distro }} + displayName: Build ${{ parameters.distro }} base image + + - script: podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io + displayName: Registry login env: # Secrets needs to be mapped as env vars to work properly QUAY_ROBOT_TOKEN: $(QUAY_ROBOT_TOKEN) + + - script: | + podman push quay.io/ansible-freeipa/upstream-tests:${{parameters.distro}}-base quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-base + displayName: Push base image + + - script: | + podman push quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server quay.io/ansible-freeipa/upstream-tests:${{ parameters.distro }}-server + displayName: Push server image diff --git a/utils/shfun b/utils/shfun new file mode 100644 index 00000000..e14b9bd6 --- /dev/null +++ b/utils/shfun @@ -0,0 +1,118 @@ +#!/bin/bash -eu +# This file is meant to be source'd by shell scripts + +SCRIPTDIR="$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}")")" + +. "${SCRIPTDIR}/shlog" + +[ -n "$(command -v python3)" ] && python="$(command -v python3)" || python="$(command -v python2)" +export python + +trap interrupt_exception SIGINT + +interrupt_exception() { + trap - SIGINT + log warn "User interrupted test execution." + # shellcheck disable=SC2119 + cleanup "${scenario:+${scenario}}" + exit 1 +} + +run_if_exists() { + cmd="${1}" + shift + [ -n "$(command -v "${cmd}")" ] && "${cmd}" "${@}" +} + +# shellcheck disable=SC2120 +cleanup() { + local container container_engine + container="${1:-${scenario:+${scenario}}}" + container_engine="${2:-${engine:-"podman"}}" + if [ "${STOP_CONTAINER:-"Y"}" == "Y" ] && [ -n "${container}" ] + then + run_if_exists stop_container "${container}" "${container_engine}" + [ -f "${inventory:-}" ] && rm "${inventory}" + else + if [ -n "${container}" ] + then + log info "Keeping container: $(${container_engine} ps --format "{{.Names}} - {{.ID}}" --filter "name=${container}")" + fi + fi + if [ "${STOP_VIRTUALENV:-"N"}" == "Y" ] + then + echo "Deactivating virtual environment" + run_if_exists deactivate + fi +} + +start_virtual_environment() { + # options -f + local FORCE_ENV VENV envdirectory + FORCE_ENV="N" + while getopts ":f" option + do + case "$option" in + f) FORCE_ENV="Y" ;; + *) die "prepare_virtual_environment: Invalid option: ${option}" ;; + esac + done + envdirectory="${test_env:-/tmp/ansible-freeipa-tests}" + + # Prepare virtual environment + VENV=$(in_python_virtualenv && echo Y || echo N) + + if [ "${FORCE_ENV}" == "Y" ] + then + run_if_exists deactivate + VENV="N" + rm -rf "$test_env" + log info "Virtual environment will be (re)created." + fi + + if [ "$VENV" == "N" ] + then + log info "Preparing virtual environment: ${envdirectory}" + if [ ! -d "${envdirectory}" ] || [ ! -f "${envdirectory}/bin/activate" ] + then + log info "Creating virtual environment: ${envdirectory}..." + log warn "RUN: ${python} -m venv ${envdirectory}" + ${python} -m venv "${envdirectory}" || die "Cannot create virtual environment." + fi + log info "Starting virtual environment: ${envdirectory}" + [ -f "${envdirectory}/bin/activate" ] || die "Failed to create virtual environment." + # shellcheck disable=SC1091 + . "${envdirectory}/bin/activate" || die "Cannot activate virtual environment." + export STOP_VIRTUALENV="Y" + log info "Installing required tools." + log none "Upgrading: pip setuptools wheel" + pip install --quiet --upgrade pip setuptools wheel + else + log info "Using current virtual environment." + fi +} + +die() { + usg="N" + if [ "${1}" == "-u" ] + then + usg="Y" + shift 1 + fi + log error "${*}" + STOP_CONTAINER="N" + cleanup "${scenario:+${scenario}}" + [ "${usg}" == "Y" ] && run_if_exists usage + exit 1 +} + +in_python_virtualenv() { + local script + read -r -d "" script </dev/null 2>&1 +} +