From 3885eb1a84759f02166c8a956e17a006da2c75a8 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Tue, 18 Aug 2020 17:38:58 -0500 Subject: [PATCH 1/6] add Makefile, downstream build script, enable CI for downstream Signed-off-by: Adam Miller --- .github/workflows/ci.yml | 91 ++++++++++++++++ Makefile | 39 +++++++ README.md | 13 ++- utils/downstream.sh | 220 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 359 insertions(+), 4 deletions(-) create mode 100644 Makefile create mode 100755 utils/downstream.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f28eef96..a11c9c59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,3 +108,94 @@ jobs: - name: Run molecule default test scenario run: molecule test working-directory: ./ansible_collections/community/kubernetes + + downstream-sanity: + runs-on: ubuntu-latest + strategy: + matrix: + python_version: ['3.6'] + steps: + - name: Check out code + uses: actions/checkout@v2 + with: + path: ansible_collections/community/kubernetes + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python_version }} + + - name: Install ansible base (devel branch) + run: pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check + + - name: Run sanity tests on Python ${{ matrix.python_version }} + run: make downstream-test-sanity + working-directory: ./ansible_collections/community/kubernetes + + downstream-integration: + runs-on: ubuntu-latest + strategy: + matrix: + python_version: ['3.6'] + steps: + - name: Check out code + uses: actions/checkout@v2 + with: + path: ansible_collections/community/kubernetes + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python_version }} + + - name: Install ansible base (devel branch) + run: pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check + + - name: Run integration tests on Python ${{ matrix.python_version }} + run: make downstream-test-integration + working-directory: ./ansible_collections/community/kubernetes + + downstream-molecule: + runs-on: ubuntu-latest + strategy: + matrix: + python_version: ['3.7'] + steps: + - name: Check out code + uses: actions/checkout@v2 + with: + path: ansible_collections/community/kubernetes + + - name: Set up KinD cluster + uses: engineerd/setup-kind@v0.4.0 + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python_version }} + + - name: Install molecule and openshift dependencies + run: pip install ansible molecule yamllint openshift flake8 + + # The latest release doesn't work with Molecule currently. + # See: https://github.com/ansible-community/molecule/issues/2757 + # - name: Install ansible base, latest release. + # run: | + # pip uninstall -y ansible + # pip install --pre ansible-base + + # The devel branch doesn't work with Molecule currently. + # See: https://github.com/ansible-community/molecule/issues/2757 + # - name: Install ansible base (devel branch) + # run: | + # pip uninstall -y ansible + # pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check + + - name: Create default collection path symlink + run: | + mkdir -p /home/runner/.ansible + ln -s /home/runner/work/kubernetes/kubernetes /home/runner/.ansible/collections + + - name: Run molecule default test scenario + run: make downstream-test-molecule + working-directory: ./ansible_collections/community/kubernetes diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..b1bb4b19 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +# Also needs to be updated in galaxy.yml +VERSION = 1.0.0 + +clean: + rm -f community-kubernetes-${VERSION}.tar.gz + rm -rf ansible_collections + +build: clean + ansible-galaxy collection build + +release: build + ansible-galaxy collection publish community-kubernetes-${VERSION}.tar.gz + +install: build + ansible-galaxy collection install -p ansible_collections community-kubernetes-${VERSION}.tar.gz + +test-sanity: + ansible-test sanity -v --docker --color $(TEST_ARGS) + +test-integration: + ansible-test integration --docker -v --color $(TEST_ARGS) + +test-molecule: + ansible-test integration --docker -v --color $(TEST_ARGS) + +downstream-test-sanity: + ./utils/downstream.sh -s + +downstream-test-integration: + ./utils/downstream.sh -i + +downstream-test-molecule: + ./utils/downstream.sh -m + +downstream-build: + ./utils/downstream.sh -b + +downstream-release: + ./utils/downstream.sh -r diff --git a/README.md b/README.md index d72d2f76..1d2f649d 100644 --- a/README.md +++ b/README.md @@ -136,15 +136,15 @@ The `tests` directory contains configuration for running sanity and integration You can run the collection's test suites with the commands: - ansible-test sanity --docker -v --color - ansible-test integration --docker -v --color + make test-sanity + make test-integration ### Testing with `molecule` -There are also integration tests in the `molecule` directory which are meant to be run against a local Kubernetes cluster, e.g. using [KinD](https://kind.sigs.k8s.io) or [Minikube](https://minikube.sigs.k8s.io). To run the tests, set up a local cluster, then run Molecule: +There are also integration tests in the `molecule` directory which are meant to be run against a local Kubernetes cluster, e.g. using [KinD](https://kind.sigs.k8s.io) or [Minikube](https://minikube.sigs.k8s.io). To setup a local cluster using KinD and run Molecule: kind create cluster - molecule test + make test-molecule ## Publishing New Versions @@ -157,6 +157,11 @@ Releases are automatically built and pushed to Ansible Galaxy for any new tag. B 1. Run `antsibull-changelog release`. 1. Commit the changes and create a PR with the changes. Wait for tests to pass, then merge it once they have. 1. Tag the version in Git and push to GitHub. + 1. Run the following commands to build and release the new version on Galaxy: + + ``` + make release + ``` After the version is published, verify it exists on the [Kubernetes Collection Galaxy page](https://galaxy.ansible.com/community/kubernetes). diff --git a/utils/downstream.sh b/utils/downstream.sh new file mode 100755 index 00000000..d13684a1 --- /dev/null +++ b/utils/downstream.sh @@ -0,0 +1,220 @@ +#!/bin/bash -eu + +# Script to dual-home the upstream and downstream Collection in a single repo +# +# This script will build or test a downstream collection, removing any +# upstream components that will not ship in the downstream release +# +# NOTES: +# - All functions are prefixed with f_ so it's obvious where they come +# from when in use throughout the script + +DOWNSTREAM_VERSION="1.0.0" +KEEP_DOWNSTREAM_TMPDIR="${KEEP_DOWNSTREAM_TMPDIR:-''}" + + +f_log_info() +{ + printf "%s:LOG:INFO: %s\n" "${0}" "${1}\n" +} + +f_prep() +{ + f_log_info "${FUNCNAME[0]}" + # Array of excluded files from downstream build (relative path) + _file_exclude=( + ) + + # Files to copy downstream (relative repo root dir path) + _file_manifest=( + CHANGELOG.rst + galaxy.yml + LICENSE + README.md + Makefile + ) + + # Directories to recursively copy downstream (relative repo root dir path) + _dir_manifest=( + changelogs + meta + plugins + tests + ) + + # Temp build dir + _tmp_dir=$(mktemp -d) + _build_dir="${_tmp_dir}/ansible_collections/kubernetes/base" + mkdir -p "${_build_dir}" +} + +f_show_help() +{ + printf "Usage: downstream.sh [OPTION]\n" + printf "\t-s\t\tCreate a temporary downstream release and perform sanity tests.\n" + printf "\t-i\t\tCreate a temporary downstream release and perform integration tests.\n" + printf "\t-m\t\tCreate a temporary downstream release and perform molecule tests.\n" + printf "\t-b\t\tCreate a downstream release and stage for release.\n" + printf "\t-r\t\tCreate a downstream release and publish release.\n" +} + +f_text_sub() +{ + # Switch FQCN and dependent components + sed -i "s/community-kubernetes/kubernetes-base/" "${_build_dir}/Makefile" + sed -i "s/community\/kubernetes/kubernetes\/base/" "${_build_dir}/Makefile" + sed -i "s/^VERSION\:/VERSION: ${DOWNSTREAM_VERSION}/" "${_build_dir}/Makefile" + sed -i "s/community.kubernetes/kubernetes.base/" "${_build_dir}/galaxy.yml" + sed -i "s/name\:.*$/name: base/" "${_build_dir}/galaxy.yml" + sed -i "s/namespace\:.*$/namespace: kubernetes/" "${_build_dir}/galaxy.yml" + sed -i "s/^version\:.*$/version: ${DOWNSTREAM_VERSION}/" "${_build_dir}/galaxy.yml" + find "${_build_dir}" -type f -exec sed -i "s/community\.kubernetes/kubernetes\.base/g" {} \; +} + +f_cleanup() +{ + f_log_info "${FUNCNAME[0]}" + if [[ -n ${KEEP_DOWNSTREAM_TMPDIR} ]]; then + if [[ -d ${_build_dir} ]]; then + rm -fr "${_build_dir}" + fi + fi +} + +# Exit and handle cleanup processes if needed +f_exit() +{ + f_cleanup + exit "$0" +} + +f_create_collection_dir_structure() +{ + f_log_info "${FUNCNAME[0]}" + # Create the Collection + for f_name in "${_file_manifest[@]}"; + do + cp "./${f_name}" "${_build_dir}/${f_name}" + done + for d_name in "${_dir_manifest[@]}"; + do + cp -r "./${d_name}" "${_build_dir}/${d_name}" + done + for exclude_file in "${_file_exclude[@]}"; + do + if [[ -f "${_build_dir}/${exclude_file}" ]]; then + rm -f "${_build_dir}/${exclude_file}" + fi + done +} + +f_copy_collection_to_working_dir() +{ + f_log_info "${FUNCNAME[0]}" + # Copy the Collection build result into original working dir + cp "${_build_dir}"/*.tar.gz ./ +} + +f_common_steps() +{ + f_log_info "${FUNCNAME[0]}" + f_prep + f_create_collection_dir_structure + f_text_sub +} + +# Run the test sanity scanerio +f_test_sanity_option() +{ + f_log_info "${FUNCNAME[0]}" + f_common_steps + pushd "${_build_dir}" || return + f_log_info "SANITY TEST PWD: ${PWD}" + make test-sanity + popd || return + f_cleanup +} + +# Run the test integration +f_test_integration_option() +{ + f_log_info "${FUNCNAME[0]}" + f_common_steps + pushd "${_build_dir}" || return + f_log_info "INTEGRATION TEST WD: ${PWD}" + make test-integration + popd || return + f_cleanup +} + +# Run the molecule tests +f_test_molecule_option() +{ + f_log_info "${FUNCNAME[0]}" + f_common_steps + pushd "${_build_dir}" || return + f_log_info "MOLECULE TEST WD: ${PWD}" + make test-molecule + popd || return + f_cleanup +} + +# Run the release scanerio +f_release_option() +{ + f_log_info "${FUNCNAME[0]}" + f_common_steps + pushd "${_build_dir}" || return + f_log_info "RELEASE WD: ${PWD}" + make release + popd || return + f_cleanup +} + +# Run the build scanerio +f_build_option() +{ + f_log_info "${FUNCNAME[0]}" + f_common_steps + pushd "${_build_dir}" || return + f_log_info "BUILD WD: ${PWD}" + make build + popd || return + f_copy_collection_to_working_dir + f_cleanup +} + +# If no options are passed, display usage and exit +if [[ "${#}" -eq "0" ]]; then + f_show_help + f_exit 0 +fi + +# Handle options +while getopts ":simrb" option +do + case $option in + s) + f_test_sanity_option + ;; + i) + f_test_integration_option + ;; + m) + f_test_molecule_option + ;; + r) + f_release_option + ;; + b) + f_build_option + ;; + *) + printf "ERROR: Unimplemented option chosen.\n" + f_show_help + f_exit 1 + ;; # Default. + esac +done + +# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 From f9fc0f7d69ec7223f35250a3c337cbb361a7d347 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 26 Aug 2020 11:33:00 -0500 Subject: [PATCH 2/6] fix README, use Makefile everywhere Signed-off-by: Adam Miller --- .github/workflows/ci.yml | 6 +++--- Makefile | 9 ++++++--- README.md | 5 ----- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a11c9c59..370669af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: run: pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check - name: Run sanity tests on Python ${{ matrix.python_version }} - run: ansible-test sanity --docker -v --color --python ${{ matrix.python_version }} + run: make test-sanity PYTHON_VERSION=${{ matrix.python_version }} working-directory: ./ansible_collections/community/kubernetes integration: @@ -53,7 +53,7 @@ jobs: run: pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check - name: Run integration tests on Python ${{ matrix.python_version }} - run: ansible-test integration --docker -v --color --retry-on-error --python ${{ matrix.python_version }} --continue-on-error --diff --coverage + run: make test-integration PYTHON_VERSION=${{ matrix.python_version }} working-directory: ./ansible_collections/community/kubernetes - name: Generate coverage report. @@ -106,7 +106,7 @@ jobs: ln -s /home/runner/work/kubernetes/kubernetes /home/runner/.ansible/collections - name: Run molecule default test scenario - run: molecule test + run: make test-molecule working-directory: ./ansible_collections/community/kubernetes downstream-sanity: diff --git a/Makefile b/Makefile index b1bb4b19..9dcd286f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ # Also needs to be updated in galaxy.yml VERSION = 1.0.0 +TEST_ARGS ?= "" +PYTHON_VERSION ?= `python -c 'import platform; print("{0}.{1}".format(platform.python_version_tuple()[0], platform.python_version_tuple()[1]))'` + clean: rm -f community-kubernetes-${VERSION}.tar.gz rm -rf ansible_collections @@ -15,13 +18,13 @@ install: build ansible-galaxy collection install -p ansible_collections community-kubernetes-${VERSION}.tar.gz test-sanity: - ansible-test sanity -v --docker --color $(TEST_ARGS) + ansible-test sanity --docker -v --color --python $(PYTHON_VERSION) $(TEST_ARGS) test-integration: - ansible-test integration --docker -v --color $(TEST_ARGS) + ansible-test integration --docker -v --color --retry-on-error --python $(PYTHON_VERSION) --continue-on-error --diff --coverage $(TEST_ARGS) test-molecule: - ansible-test integration --docker -v --color $(TEST_ARGS) + molecule test downstream-test-sanity: ./utils/downstream.sh -s diff --git a/README.md b/README.md index 1d2f649d..e3d7c6b9 100644 --- a/README.md +++ b/README.md @@ -157,11 +157,6 @@ Releases are automatically built and pushed to Ansible Galaxy for any new tag. B 1. Run `antsibull-changelog release`. 1. Commit the changes and create a PR with the changes. Wait for tests to pass, then merge it once they have. 1. Tag the version in Git and push to GitHub. - 1. Run the following commands to build and release the new version on Galaxy: - - ``` - make release - ``` After the version is published, verify it exists on the [Kubernetes Collection Galaxy page](https://galaxy.ansible.com/community/kubernetes). From b5eba332c9a7051358717fe6a08491591c6caf79 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 26 Aug 2020 11:49:50 -0500 Subject: [PATCH 3/6] fix makefile Signed-off-by: Adam Miller --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9dcd286f..a6f0938c 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ PYTHON_VERSION ?= `python -c 'import platform; print("{0}.{1}".format(platform.p clean: rm -f community-kubernetes-${VERSION}.tar.gz rm -rf ansible_collections + rm -rf tests/output build: clean ansible-galaxy collection build @@ -18,10 +19,10 @@ install: build ansible-galaxy collection install -p ansible_collections community-kubernetes-${VERSION}.tar.gz test-sanity: - ansible-test sanity --docker -v --color --python $(PYTHON_VERSION) $(TEST_ARGS) + ansible-test sanity --docker -v --color --python $(PYTHON_VERSION) $(?TEST_ARGS) test-integration: - ansible-test integration --docker -v --color --retry-on-error --python $(PYTHON_VERSION) --continue-on-error --diff --coverage $(TEST_ARGS) + ansible-test integration --docker -v --color --retry-on-error --python $(PYTHON_VERSION) --continue-on-error --diff --coverage $(?TEST_ARGS) test-molecule: molecule test From f2053480912d5920d531c24c5ead3ceeeeb4fb20 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 26 Aug 2020 12:55:56 -0500 Subject: [PATCH 4/6] fix downstream molecule tests Signed-off-by: Adam Miller --- utils/downstream.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/downstream.sh b/utils/downstream.sh index d13684a1..14348713 100755 --- a/utils/downstream.sh +++ b/utils/downstream.sh @@ -32,6 +32,7 @@ f_prep() LICENSE README.md Makefile + .yamllint ) # Directories to recursively copy downstream (relative repo root dir path) @@ -40,6 +41,7 @@ f_prep() meta plugins tests + molecule ) # Temp build dir From ec74038a1e41e764a157738f53f05217b7a4ebc0 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 26 Aug 2020 13:07:14 -0500 Subject: [PATCH 5/6] need flake8 setup.cfg for downstream tests also Signed-off-by: Adam Miller --- utils/downstream.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/downstream.sh b/utils/downstream.sh index 14348713..96fda78f 100755 --- a/utils/downstream.sh +++ b/utils/downstream.sh @@ -32,6 +32,7 @@ f_prep() LICENSE README.md Makefile + setup.cfg .yamllint ) From 8c7ea120245911e507b277ec8312ad40451abda2 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 26 Aug 2020 15:07:21 -0500 Subject: [PATCH 6/6] only test downstream against Ansible 2.9.x Signed-off-by: Adam Miller --- .github/workflows/ci.yml | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 370669af..3971bc4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: run: make test-molecule working-directory: ./ansible_collections/community/kubernetes - downstream-sanity: + downstream-sanity-2.9: runs-on: ubuntu-latest strategy: matrix: @@ -126,13 +126,13 @@ jobs: python-version: ${{ matrix.python_version }} - name: Install ansible base (devel branch) - run: pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check + run: pip install "ansible>=2.9.0,<2.10.0" - name: Run sanity tests on Python ${{ matrix.python_version }} run: make downstream-test-sanity working-directory: ./ansible_collections/community/kubernetes - downstream-integration: + downstream-integration-2.9: runs-on: ubuntu-latest strategy: matrix: @@ -149,13 +149,13 @@ jobs: python-version: ${{ matrix.python_version }} - name: Install ansible base (devel branch) - run: pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check + run: pip install "ansible>=2.9.0,<2.10.0" - name: Run integration tests on Python ${{ matrix.python_version }} run: make downstream-test-integration working-directory: ./ansible_collections/community/kubernetes - downstream-molecule: + downstream-molecule-2.9: runs-on: ubuntu-latest strategy: matrix: @@ -175,21 +175,7 @@ jobs: python-version: ${{ matrix.python_version }} - name: Install molecule and openshift dependencies - run: pip install ansible molecule yamllint openshift flake8 - - # The latest release doesn't work with Molecule currently. - # See: https://github.com/ansible-community/molecule/issues/2757 - # - name: Install ansible base, latest release. - # run: | - # pip uninstall -y ansible - # pip install --pre ansible-base - - # The devel branch doesn't work with Molecule currently. - # See: https://github.com/ansible-community/molecule/issues/2757 - # - name: Install ansible base (devel branch) - # run: | - # pip uninstall -y ansible - # pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check + run: pip install "ansible>=2.9.0,<2.10.0" molecule yamllint openshift flake8 - name: Create default collection path symlink run: |