diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 59aaa757..38200930 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,17 +37,11 @@ jobs: run: | ansible-galaxy collection install community.general kubernetes.core:1.2.1 operator_sdk.util - - name: Setup Minikube - uses: manusa/actions-setup-minikube@v2.4.2 - with: - minikube version: 'v1.16.0' - kubernetes version: 'v1.19.2' - github token: ${{ secrets.GITHUB_TOKEN }} - - name: Run Molecule env: MOLECULE_VERBOSITY: 3 PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' run: | - molecule test -s test-minikube + make kustomize + KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind diff --git a/.yamllint b/.yamllint index 01becb8a..e79e4dd6 100644 --- a/.yamllint +++ b/.yamllint @@ -4,6 +4,7 @@ extends: default ignore: | .cache/ deploy/olm-catalog + config/testing/kustomization.yaml rules: truthy: disable diff --git a/config/samples/awx_v1alpha1_awx.yaml b/config/samples/awx_v1alpha1_awx.yaml deleted file mode 100644 index b757761a..00000000 --- a/config/samples/awx_v1alpha1_awx.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: awx.ansible.com/v1beta1 -kind: AWX -metadata: - name: awx-sample -spec: - # Add fields here - foo: bar diff --git a/config/samples/awx_v1beta1_awx.yaml b/config/samples/awx_v1beta1_awx.yaml new file mode 100644 index 00000000..1e7b1d8c --- /dev/null +++ b/config/samples/awx_v1beta1_awx.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: awx.ansible.com/v1beta1 +kind: AWX +metadata: + name: example-awx +spec: + service_account_annotations: | + foo: bar + deployment_type: awx + ingress_type: ingress + web_resource_requirements: + requests: + cpu: 500m + memory: 128M + task_resource_requirements: + requests: + cpu: 500m + memory: 128M + ee_resource_requirements: + requests: + cpu: 200m + memory: 64M diff --git a/config/testing/kustomization.yaml b/config/testing/kustomization.yaml index 4bb58082..b6b45328 100644 --- a/config/testing/kustomization.yaml +++ b/config/testing/kustomization.yaml @@ -1,4 +1,3 @@ ---- # Adds namespace to all resources. namespace: osdk-test namePrefix: osdk- @@ -6,15 +5,17 @@ namePrefix: osdk- # commonLabels: # someName: someValue patchesStrategicMerge: - - manager_image.yaml - - debug_logs_patch.yaml - - ../default/manager_auth_proxy_patch.yaml +- manager_image.yaml +- debug_logs_patch.yaml +- ../default/manager_auth_proxy_patch.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../crd - - ../rbac - - ../manager +- ../crd +- ../rbac +- ../manager images: - - name: testing - newName: testing-operator +- name: testing + newName: testing-operator +patches: +- path: pull_policy/Never.yaml diff --git a/molecule/default/asserts.yml b/molecule/default/asserts.yml deleted file mode 100644 index b2cf31a8..00000000 --- a/molecule/default/asserts.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Verify cluster resources - hosts: localhost - connection: local - - vars: - ansible_python_interpreter: '{{ ansible_playbook_python }}' - - tasks: - - name: Get AWX Kind data - k8s_info: - api_version: awx.ansible.com/v1beta1 - kind: AWX - namespace: example-awx - label_selectors: - - "app.kubernetes.io/name=example-awx" - - "app.kubernetes.io/part-of=example-awx" - - "app.kubernetes.io/managed-by=awx-operator" - - "app.kubernetes.io/component=awx" - register: awx_kind - - - name: Verify there is one AWX kind - assert: - that: '{{ (awx_kind.resources | length) == 1 }}' - - - name: Get AWX Pod data - k8s_info: - kind: Pod - namespace: example-awx - label_selectors: - - "app.kubernetes.io/name=example-awx" - - "app.kubernetes.io/part-of=example-awx" - - "app.kubernetes.io/managed-by=awx-operator" - - "app.kubernetes.io/component=awx" - register: tower_pods - - - name: Verify there is one AWX pod - assert: - that: '{{ (tower_pods.resources | length) == 1 }}' diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 4a8b251e..0633db98 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -2,9 +2,17 @@ - name: Converge hosts: localhost connection: local - vars: - ansible_python_interpreter: '{{ ansible_playbook_python }}' - roles: - - installer + gather_facts: no + collections: + - community.kubernetes -- import_playbook: '{{ playbook_dir }}/asserts.yml' + tasks: + - name: Create Namespace + k8s: + api_version: v1 + kind: Namespace + name: '{{ namespace }}' + + - import_tasks: kustomize.yml + vars: + state: present diff --git a/molecule/default/create.yml b/molecule/default/create.yml new file mode 100644 index 00000000..1eeaf922 --- /dev/null +++ b/molecule/default/create.yml @@ -0,0 +1,6 @@ +--- +- name: Create + hosts: localhost + connection: local + gather_facts: false + tasks: [] diff --git a/molecule/default/destroy.yml b/molecule/default/destroy.yml new file mode 100644 index 00000000..af55e4f1 --- /dev/null +++ b/molecule/default/destroy.yml @@ -0,0 +1,24 @@ +--- +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + collections: + - community.kubernetes + + tasks: + - import_tasks: kustomize.yml + vars: + state: absent + + - name: Destroy Namespace + k8s: + api_version: v1 + kind: Namespace + name: '{{ namespace }}' + state: absent + + - name: Unset pull policy + command: '{{ kustomize }} edit remove patch pull_policy/{{ operator_pull_policy }}.yaml' + args: + chdir: '{{ config_dir }}/testing' diff --git a/molecule/default/kustomize.yml b/molecule/default/kustomize.yml new file mode 100644 index 00000000..f3d888c2 --- /dev/null +++ b/molecule/default/kustomize.yml @@ -0,0 +1,15 @@ +--- +- name: Build kustomize testing overlay + # load_restrictor must be set to none so we can load patch files from the default overlay + command: '{{ kustomize }} build --load_restrictor none .' + args: + chdir: '{{ config_dir }}/testing' + register: resources + changed_when: false + +- name: Set resources to {{ state }} + k8s: + definition: '{{ item }}' + state: '{{ state }}' + wait: yes + loop: '{{ resources.stdout | from_yaml_all | list }}' diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index c110f12a..09621b73 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -2,28 +2,35 @@ dependency: name: galaxy driver: - name: docker + name: delegated lint: | set -e yamllint . - ansible-lint platforms: - - name: kind-default + - name: cluster groups: - k8s - image: bsycorp/kind:latest-1.14 - privileged: True - override_command: no - exposed_ports: - - 8443/tcp - - 10080/tcp - published_ports: - - 0.0.0.0:${TEST_CLUSTER_PORT:-9443}:8443/tcp - pre_build_image: yes provisioner: name: ansible - log: True + lint: | + set -e + ansible-lint inventory: group_vars: all: - operator_namespace: ${TEST_NAMESPACE:-example-awx} + namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test} + host_vars: + localhost: + ansible_python_interpreter: '{{ ansible_playbook_python }}' + config_dir: ${MOLECULE_PROJECT_DIRECTORY}/config + samples_dir: ${MOLECULE_PROJECT_DIRECTORY}/config/samples + operator_image: ${OPERATOR_IMAGE:-""} + operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"} + kustomize: ${KUSTOMIZE_PATH:-kustomize} + env: + K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"} +verifier: + name: ansible + lint: | + set -e + ansible-lint diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index 118388ea..ed40370c 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -1,39 +1,28 @@ --- -- name: Prepare operator resources +- name: Prepare hosts: localhost connection: local - - vars: - ansible_python_interpreter: '{{ ansible_playbook_python }}' - deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" - templates_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/ansible/templates" - vars_files: - - "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/ansible/group_vars/all" + gather_facts: false tasks: - - name: Create AWX Custom Resource Definition - k8s: - definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_crd.yaml'])) }}" + - name: Ensure operator image is set + fail: + msg: | + You must specify the OPERATOR_IMAGE environment variable in order to run the + 'default' scenario + when: not operator_image - - name: Create AWXBackup Custom Resource Definition - k8s: - definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxbackup_v1beta1_crd.yaml'])) }}" + - name: Set testing image + command: '{{ kustomize }} edit set image testing={{ operator_image }}' + args: + chdir: '{{ config_dir }}/testing' - - name: Create AWXRestore Custom Resource Definition - k8s: - definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxrestore_v1beta1_crd.yaml'])) }}" + - name: Set pull policy + command: '{{ kustomize }} edit add patch --path pull_policy/{{ operator_pull_policy }}.yaml' + args: + chdir: '{{ config_dir }}/testing' - - name: Ensure specified namespace is present - k8s: - api_version: v1 - kind: Namespace - name: '{{ operator_namespace }}' - - - name: Create RBAC resources - k8s: - definition: "{{ lookup('template', '/'.join([templates_dir, item])) }}" - namespace: '{{ operator_namespace }}' - with_items: - - role.yml.j2 - - role_binding.yml.j2 - - service_account.yml.j2 + - name: Set testing namespace + command: '{{ kustomize }} edit set namespace {{ namespace }}' + args: + chdir: '{{ config_dir }}/testing' diff --git a/molecule/default/tasks/awx_test.yml b/molecule/default/tasks/awx_test.yml new file mode 100644 index 00000000..66cef064 --- /dev/null +++ b/molecule/default/tasks/awx_test.yml @@ -0,0 +1,19 @@ +--- +- name: Create the awx.ansible.com/v1alpha1.AWX + k8s: + state: present + namespace: '{{ namespace }}' + definition: "{{ lookup('template', '/'.join([samples_dir, cr_file])) | from_yaml }}" + wait: yes + wait_timeout: 900 + wait_condition: + type: Running + reason: Successful + status: "True" + vars: + cr_file: 'awx_v1beta1_awx.yaml' + +# - name: Add assertions here +# assert: +# that: false +# fail_msg: FIXME Add real assertions for your operator diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml new file mode 100644 index 00000000..cc28aacf --- /dev/null +++ b/molecule/default/verify.yml @@ -0,0 +1,57 @@ +--- +- name: Verify + hosts: localhost + connection: local + gather_facts: no + collections: + - community.kubernetes + + vars: + ctrl_label: control-plane=controller-manager + + tasks: + - block: + - name: Import all test files from tasks/ + include_tasks: '{{ item }}' + with_fileglob: + - tasks/*_test.yml + rescue: + - name: Retrieve relevant resources + k8s_info: + api_version: '{{ item.api_version }}' + kind: '{{ item.kind }}' + namespace: '{{ namespace }}' + loop: + - api_version: v1 + kind: Pod + - api_version: apps/v1 + kind: Deployment + - api_version: v1 + kind: Secret + - api_version: v1 + kind: ConfigMap + register: debug_resources + + - name: Retrieve Pod logs + k8s_log: + name: '{{ item.metadata.name }}' + namespace: '{{ namespace }}' + container: manager + loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}" + register: debug_logs + + - name: Output gathered resources + debug: + var: debug_resources + + - name: Output gathered logs + debug: + var: item.log_lines + loop: '{{ debug_logs.results }}' + + - name: Re-emit failure + vars: + failed_task: + result: '{{ ansible_failed_result }}' + fail: + msg: '{{ failed_task }}' diff --git a/molecule/kind/converge.yml b/molecule/kind/converge.yml new file mode 100644 index 00000000..8bd5700f --- /dev/null +++ b/molecule/kind/converge.yml @@ -0,0 +1,24 @@ +--- +- name: Converge + hosts: localhost + connection: local + gather_facts: no + + tasks: + - name: Build operator image + docker_image: + build: + path: '{{ project_dir }}' + pull: no + name: '{{ operator_image }}' + tag: latest + push: no + source: build + force_source: yes + + - name: Load image into kind cluster + command: kind load docker-image --name osdk-test '{{ operator_image }}' + register: result + changed_when: '"not yet present" in result.stdout' + +- import_playbook: ../default/converge.yml diff --git a/molecule/kind/create.yml b/molecule/kind/create.yml new file mode 100644 index 00000000..66a84a14 --- /dev/null +++ b/molecule/kind/create.yml @@ -0,0 +1,8 @@ +--- +- name: Create + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Create test kind cluster + command: kind create cluster --name osdk-test --kubeconfig {{ kubeconfig }} diff --git a/molecule/kind/destroy.yml b/molecule/kind/destroy.yml new file mode 100644 index 00000000..a1c3eac9 --- /dev/null +++ b/molecule/kind/destroy.yml @@ -0,0 +1,16 @@ +--- +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + collections: + - community.kubernetes + + tasks: + - name: Destroy test kind cluster + command: kind delete cluster --name osdk-test --kubeconfig {{ kubeconfig }} + + - name: Unset pull policy + command: '{{ kustomize }} edit remove patch pull_policy/{{ operator_pull_policy }}.yaml' + args: + chdir: '{{ config_dir }}/testing' diff --git a/molecule/kind/molecule.yml b/molecule/kind/molecule.yml new file mode 100644 index 00000000..80a18d39 --- /dev/null +++ b/molecule/kind/molecule.yml @@ -0,0 +1,42 @@ +--- +dependency: + name: galaxy +driver: + name: delegated +lint: | + set -e + yamllint . +platforms: + - name: cluster + groups: + - k8s +provisioner: + name: ansible + playbooks: + prepare: ../default/prepare.yml + verify: ../default/verify.yml + lint: | + set -e + ansible-lint + inventory: + group_vars: + all: + namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test} + host_vars: + localhost: + ansible_python_interpreter: '{{ ansible_playbook_python }}' + config_dir: ${MOLECULE_PROJECT_DIRECTORY}/config + samples_dir: ${MOLECULE_PROJECT_DIRECTORY}/config/samples + project_dir: ${MOLECULE_PROJECT_DIRECTORY} + operator_image: testing-operator + operator_pull_policy: "Never" + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + kustomize: ${KUSTOMIZE_PATH:-kustomize} + env: + K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig + KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig +verifier: + name: ansible + lint: | + set -e + ansible-lint diff --git a/molecule/test-minikube/converge.yml b/molecule/test-minikube/converge.yml deleted file mode 100644 index 8127b7e2..00000000 --- a/molecule/test-minikube/converge.yml +++ /dev/null @@ -1,140 +0,0 @@ ---- -# TODO: For some reason prepare is not run after a destroy in the Minikube env. -- import_playbook: ../default/prepare.yml - -- name: Build Operator in Minikube - hosts: localhost - connection: local - - vars: - image_name: awx.ansible.com/awx-operator:testing - - tasks: - # Use raw Docker commands inside Minikube to avoid extra Python dependencies. - - name: Get existing image hash - shell: | - eval $(minikube docker-env) - docker images -q {{ image_name }} - register: prev_hash - changed_when: false - - - name: Build Operator Image - shell: | - eval $(minikube docker-env) - docker build -f ../../build/Dockerfile -t {{ image_name }} ../.. - register: build_cmd - changed_when: not prev_hash.stdout or (prev_hash.stdout and prev_hash.stdout not in ''.join(build_cmd.stdout_lines[-2:])) - -- name: Converge - hosts: localhost - connection: local - - vars: - ansible_python_interpreter: '{{ ansible_playbook_python }}' - deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" - templates_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/ansible/templates" - pull_policy: Never - operator_image: awx.ansible.com/awx-operator - operator_version: testing - ansible_debug_logs: "true" - # Change this to _awx to test AWX, _tower to test Tower. - custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_molecule.yaml'])) | from_yaml }}" - - tasks: - - block: - - name: Delete the Operator Deployment - k8s: - state: absent - namespace: '{{ operator_namespace }}' - definition: "{{ lookup('template', '/'.join([templates_dir, 'operator.yml.j2'])) }}" - register: delete_deployment - when: build_cmd.changed - - - name: Wait 30s for Operator Deployment to terminate - k8s_info: - api_version: '{{ definition.apiVersion }}' - kind: '{{ definition.kind }}' - namespace: '{{ operator_namespace }}' - name: '{{ definition.metadata.name }}' - vars: - definition: "{{ lookup('template', '/'.join([templates_dir, 'operator.yml.j2'])) | from_yaml }}" - register: deployment - until: not deployment.resources - delay: 3 - retries: 10 - when: delete_deployment.changed - - - name: Create the Operator Deployment - k8s: - namespace: '{{ operator_namespace }}' - definition: "{{ lookup('template', '/'.join([templates_dir, 'operator.yml.j2'])) }}" - - - name: Ensure the AWX custom_resource namespace exists - k8s: - state: present - name: '{{ custom_resource.metadata.namespace }}' - kind: Namespace - api_version: v1 - - - name: Create the AWX Custom Resource - k8s: - state: present - namespace: '{{ custom_resource.metadata.namespace }}' - definition: '{{ custom_resource }}' - - - name: Wait 15m for reconciliation to run - k8s_info: - api_version: '{{ custom_resource.apiVersion }}' - kind: '{{ custom_resource.kind }}' - namespace: '{{ custom_resource.metadata.namespace }}' - name: '{{ custom_resource.metadata.name }}' - register: cr - until: - - "'Successful' in (cr | json_query('resources[].status.conditions[].reason'))" - delay: 6 - retries: 150 - - rescue: - - - name: debug cr - ignore_errors: yes # noqa ignore-errors - failed_when: false - debug: - var: debug_cr - vars: - debug_cr: '{{ lookup("k8s", - kind=custom_resource.kind, - api_version=custom_resource.apiVersion, - namespace=custom_resource.metadata.namespace, - resource_name=custom_resource.metadata.name) - }}' - - - name: debug awx deployment - ignore_errors: yes # noqa ignore-errors - failed_when: false - debug: - var: deploy - vars: - deploy: '{{ lookup("k8s", - kind="Deployment", - api_version="apps/v1", - namespace=custom_resource.metadata.namespace, - label_selector="app.kubernetes.io/name=example-awx") - }}' - - - name: get operator logs - ignore_errors: yes - failed_when: false - command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ operator_namespace }} -c operator - vars: - definition: "{{ lookup('template', '/'.join([templates_dir, 'operator.yml.j2'])) | from_yaml }}" - register: log - - - name: print debug output - debug: var=log.stdout_lines - - - name: fail if converge didn't succeed - fail: - msg: "Failed on action: converge" - -- import_playbook: '{{ playbook_dir }}/../default/asserts.yml' diff --git a/molecule/test-minikube/molecule.yml b/molecule/test-minikube/molecule.yml deleted file mode 100644 index bde1c276..00000000 --- a/molecule/test-minikube/molecule.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: delegated - options: - managed: False - ansible_connection_options: {} -lint: | - set -e - yamllint . - ansible-lint -platforms: - - name: test-minikube - groups: - - k8s -provisioner: - name: ansible - inventory: - group_vars: - all: - operator_namespace: ${TEST_NAMESPACE:-example-awx} - env: - ANSIBLE_ROLES_PATH: ${MOLECULE_PROJECT_DIRECTORY}/roles -scenario: - test_sequence: - - lint - - destroy - - dependency - - syntax - - create - - prepare - - converge - - destroy diff --git a/molecule/test-minikube/prepare.yml b/molecule/test-minikube/prepare.yml deleted file mode 100644 index 8ca82682..00000000 --- a/molecule/test-minikube/prepare.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -- import_playbook: ../default/prepare.yml