From 2c527990fa9b3a8311f070c950e3863bdc39bb2a Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Wed, 6 Nov 2019 16:24:11 -0600 Subject: [PATCH] Issue #1: Get a minikube configuration working for local Molecule testing. --- README.md | 15 +- build/test-framework/ansible-test.sh | 2 +- deploy/operator.yaml | 4 +- molecule/default/asserts.yml | 9 +- molecule/default/prepare.yml | 53 +++---- molecule/test-cluster/playbook.yml | 33 ----- molecule/test-local/playbook.yml | 82 ++++++----- molecule/test-local/prepare.yml | 55 +++++--- .../molecule.yml | 6 +- molecule/test-minikube/playbook.yml | 133 ++++++++++++++++++ molecule/test-minikube/prepare.yml | 2 + 11 files changed, 254 insertions(+), 140 deletions(-) delete mode 100644 molecule/test-cluster/playbook.yml rename molecule/{test-cluster => test-minikube}/molecule.yml (85%) create mode 100644 molecule/test-minikube/playbook.yml create mode 100644 molecule/test-minikube/prepare.yml diff --git a/README.md b/README.md index 9372a4be..a4f557e7 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,17 @@ [![Build Status](https://travis-ci.com/geerlingguy/tower-operator.svg?branch=master)](https://travis-ci.com/geerlingguy/tower-operator) -A Tower operator for Kubernetes built with Operator SDK. +An [Ansible Tower](https://www.ansible.com/products/tower) operator for Kubernetes built with [Operator SDK](https://github.com/operator-framework/operator-sdk) and Ansible. + +## Testing + +This Operator includes a molecule-based test framework, which can be executed standalone in Docker (e.g. in CI or in a single Docker container anywhere), or inside any kind of Kubernetes cluster (e.g. Minikube). + +### Testing in Docker (standalone) + + 1. `molecule converge -s test-local` + +### Testing in Minikube + + 1. `minikube start` + 1. `molecule converge -s test-minikube` diff --git a/build/test-framework/ansible-test.sh b/build/test-framework/ansible-test.sh index 9719f260..e562ec5b 100644 --- a/build/test-framework/ansible-test.sh +++ b/build/test-framework/ansible-test.sh @@ -4,4 +4,4 @@ export WATCH_NAMESPACE=${TEST_NAMESPACE} trap "kill $!" SIGINT SIGTERM EXIT cd ${HOME}/project -exec molecule test -s test-cluster +exec molecule test -s test-minikube diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 8f1c8267..44cb9f4f 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -41,8 +41,8 @@ spec: fieldPath: metadata.name - name: OPERATOR_NAME value: tower-operator - - name: ANSIBLE_VERBOSITY - value: '3' + # - name: ANSIBLE_VERBOSITY + # value: '3' volumes: - name: runner emptyDir: {} diff --git a/molecule/default/asserts.yml b/molecule/default/asserts.yml index c1f2b8de..0fc71c7b 100644 --- a/molecule/default/asserts.yml +++ b/molecule/default/asserts.yml @@ -8,7 +8,7 @@ tasks: - name: Get tower Pod data - k8s_facts: + k8s_info: kind: Pod namespace: example-tower label_selectors: @@ -18,10 +18,3 @@ - name: Verify there is one tower pod assert: that: '{{ (tower_pods.resources | length) == 1 }}' - -- name: Verify tower functionality - hosts: k8s - - vars: [] - - tasks: [] diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index e7d24301..6aecea32 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -1,35 +1,28 @@ --- -- name: Prepare - hosts: k8s - gather_facts: no +- name: Prepare operator resources + hosts: localhost + connection: local + vars: - kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + ansible_python_interpreter: '{{ ansible_playbook_python }}' + deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" + tasks: - - name: delete the kubeconfig if present - file: - path: '{{ kubeconfig }}' - state: absent - delegate_to: localhost + - name: Create Custom Resource Definition + k8s: + definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/tower_v1alpha1_tower_crd.yaml'])) }}" - - name: Fetch the kubeconfig - fetch: - dest: '{{ kubeconfig }}' - flat: yes - src: /root/.kube/config + - name: Ensure specified namespace is present + k8s: + api_version: v1 + kind: Namespace + name: '{{ operator_namespace }}' - - name: Change the kubeconfig port to the proper value - replace: - regexp: 8443 - replace: "{{ lookup('env', 'KIND_PORT') }}" - path: '{{ kubeconfig }}' - delegate_to: localhost - - - name: Wait for the Kubernetes API to become available (this could take a minute) - uri: - url: "http://localhost:10080/kubernetes-ready" - status_code: 200 - validate_certs: no - register: result - until: (result.status|default(-1)) == 200 - retries: 60 - delay: 5 + - name: Create RBAC resources + k8s: + definition: "{{ lookup('template', '/'.join([deploy_dir, item])) }}" + namespace: '{{ operator_namespace }}' + with_items: + - role.yaml + - role_binding.yaml + - service_account.yaml diff --git a/molecule/test-cluster/playbook.yml b/molecule/test-cluster/playbook.yml deleted file mode 100644 index 7100a472..00000000 --- a/molecule/test-cluster/playbook.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: Converge - hosts: localhost - connection: local - vars: - ansible_python_interpreter: '{{ ansible_playbook_python }}' - deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" - image_name: tower.ansible.com/tower-operator:testing - custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/tower_v1alpha1_tower_cr.yaml'])) | from_yaml }}" - tasks: - - name: Create the tower.ansible.com/v1alpha1.Tower - k8s: - namespace: '{{ namespace }}' - definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/tower_v1alpha1_tower_cr.yaml'])) }}" - - - name: Get the newly created Custom Resource - debug: - msg: "{{ lookup('k8s', group='tower.ansible.com', api_version='v1alpha1', kind='Tower', namespace=namespace, resource_name=custom_resource.metadata.name) }}" - - - name: Wait 60s for reconciliation to run - k8s_facts: - api_version: 'v1alpha1' - kind: 'Tower' - namespace: '{{ namespace }}' - name: '{{ custom_resource.metadata.name }}' - register: reconcile_cr - until: - - "'Successful' in (reconcile_cr | json_query('resources[].status.conditions[].reason'))" - delay: 6 - retries: 10 - -- import_playbook: '{{ playbook_dir }}/../default/asserts.yml' diff --git a/molecule/test-local/playbook.yml b/molecule/test-local/playbook.yml index d77bf845..51367183 100644 --- a/molecule/test-local/playbook.yml +++ b/molecule/test-local/playbook.yml @@ -1,5 +1,5 @@ --- -- name: Build Operator in Kubernetes docker container +- name: Build Operator in Kind container hosts: k8s vars: @@ -29,7 +29,9 @@ custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/tower_v1alpha1_tower_cr.yaml'])) | from_yaml }}" tasks: + - block: + - name: Delete the Operator Deployment k8s: state: absent @@ -39,7 +41,7 @@ when: hostvars[groups.k8s.0].build_cmd.changed - name: Wait 30s for Operator Deployment to terminate - k8s_facts: + k8s_info: api_version: '{{ definition.apiVersion }}' kind: '{{ definition.kind }}' namespace: '{{ operator_namespace }}' @@ -71,7 +73,7 @@ definition: '{{ custom_resource }}' - name: Wait 5m for reconciliation to run - k8s_facts: + k8s_info: api_version: '{{ custom_resource.apiVersion }}' kind: '{{ custom_resource.kind }}' namespace: '{{ custom_resource.metadata.namespace }}' @@ -81,46 +83,48 @@ - "'Successful' in (cr | json_query('resources[].status.conditions[].reason'))" delay: 6 retries: 50 + rescue: - - name: debug cr - ignore_errors: yes - failed_when: false - debug: - var: debug_cr - vars: - debug_cr: '{{ lookup("k8s", - kind=custom_resource.kind, - api_version=custom_resource.apiVersion, - namespace=custom_resource.metadata.namespace, - resource_name=custom_resource.metadata.name - )}}' - - name: debug tower deployment - ignore_errors: yes - failed_when: false - debug: - var: deploy - vars: - deploy: '{{ lookup("k8s", - kind="Deployment", - api_version="apps/v1", - namespace=custom_resource.metadata.namespace, - label_selector="app=tower" - )}}' + - name: debug cr + ignore_errors: yes + failed_when: false + debug: + var: debug_cr + vars: + debug_cr: '{{ lookup("k8s", + kind=custom_resource.kind, + api_version=custom_resource.apiVersion, + namespace=custom_resource.metadata.namespace, + resource_name=custom_resource.metadata.name + )}}' - - name: get operator logs - ignore_errors: yes - failed_when: false - command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ operator_namespace }} -c operator - environment: - KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}' - vars: - definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" - register: log + - name: debug tower deployment + ignore_errors: yes + failed_when: false + debug: + var: deploy + vars: + deploy: '{{ lookup("k8s", + kind="Deployment", + api_version="apps/v1", + namespace=custom_resource.metadata.namespace, + label_selector="app=tower" + )}}' - - debug: var=log.stdout_lines + - name: get operator logs + ignore_errors: yes + failed_when: false + command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ operator_namespace }} -c operator + environment: + KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}' + vars: + definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" + register: log - - fail: - msg: "Failed on action: converge" + - debug: var=log.stdout_lines + + - fail: + msg: "Failed on action: converge" - import_playbook: '{{ playbook_dir }}/../default/asserts.yml' diff --git a/molecule/test-local/prepare.yml b/molecule/test-local/prepare.yml index ad2419c1..506741b2 100644 --- a/molecule/test-local/prepare.yml +++ b/molecule/test-local/prepare.yml @@ -1,28 +1,37 @@ --- -- import_playbook: ../default/prepare.yml - -- name: Prepare operator resources - hosts: localhost - connection: local +- name: Prepare kubernetes environment + hosts: k8s + gather_facts: no vars: - ansible_python_interpreter: '{{ ansible_playbook_python }}' - deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" tasks: - - name: Create Custom Resource Definition - k8s: - definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/tower_v1alpha1_tower_crd.yaml'])) }}" + - name: delete the kubeconfig if present + file: + path: '{{ kubeconfig }}' + state: absent + delegate_to: localhost - - name: Ensure specified namespace is present - k8s: - api_version: v1 - kind: Namespace - name: '{{ operator_namespace }}' + - name: Fetch the kubeconfig + fetch: + dest: '{{ kubeconfig }}' + flat: yes + src: /root/.kube/config - - name: Create RBAC resources - k8s: - definition: "{{ lookup('template', '/'.join([deploy_dir, item])) }}" - namespace: '{{ operator_namespace }}' - with_items: - - role.yaml - - role_binding.yaml - - service_account.yaml + - name: Change the kubeconfig port to the proper value + replace: + regexp: 8443 + replace: "{{ lookup('env', 'KIND_PORT') }}" + path: '{{ kubeconfig }}' + delegate_to: localhost + + - name: Wait for the Kubernetes API to become available (this could take a minute) + uri: + url: "http://localhost:10080/kubernetes-ready" + status_code: 200 + validate_certs: no + register: result + until: (result.status|default(-1)) == 200 + retries: 60 + delay: 5 + +- import_playbook: ../default/prepare.yml diff --git a/molecule/test-cluster/molecule.yml b/molecule/test-minikube/molecule.yml similarity index 85% rename from molecule/test-cluster/molecule.yml rename to molecule/test-minikube/molecule.yml index c4d6b9f4..b43809f3 100644 --- a/molecule/test-cluster/molecule.yml +++ b/molecule/test-minikube/molecule.yml @@ -10,7 +10,7 @@ lint: name: yamllint enabled: False platforms: -- name: test-cluster +- name: test-minikube groups: - k8s provisioner: @@ -18,14 +18,14 @@ provisioner: inventory: group_vars: all: - namespace: ${TEST_NAMESPACE:-default} + operator_namespace: ${TEST_NAMESPACE:-default} lint: name: ansible-lint enabled: False env: ANSIBLE_ROLES_PATH: ${MOLECULE_PROJECT_DIRECTORY}/roles scenario: - name: test-cluster + name: test-minikube test_sequence: - lint - destroy diff --git a/molecule/test-minikube/playbook.yml b/molecule/test-minikube/playbook.yml new file mode 100644 index 00000000..7b50e37b --- /dev/null +++ b/molecule/test-minikube/playbook.yml @@ -0,0 +1,133 @@ +--- +- name: Build Operator in Minikube + hosts: localhost + connection: local + + vars: + image_name: tower.ansible.com/tower-operator:testing + + tasks: + # Use raw Docker commands inside Minikube to avoid extra Python dependencies. + - name: Get existing image hash + shell: | + eval $(minikube docker-env) + docker images -q {{ image_name }} + register: prev_hash + changed_when: false + + - name: Build Operator Image + shell: | + eval $(minikube docker-env) + docker build -f ../../build/Dockerfile -t {{ image_name }} ../.. + register: build_cmd + changed_when: not prev_hash.stdout or (prev_hash.stdout and prev_hash.stdout not in ''.join(build_cmd.stdout_lines[-2:])) + +- name: Converge + hosts: localhost + connection: local + + vars: + ansible_python_interpreter: '{{ ansible_playbook_python }}' + deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" + pull_policy: Never + operator_image: tower.ansible.com/tower-operator:testing + custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/tower_v1alpha1_tower_cr.yaml'])) | from_yaml }}" + + tasks: + - block: + - name: Delete the Operator Deployment + k8s: + state: absent + namespace: '{{ operator_namespace }}' + definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) }}" + register: delete_deployment + when: build_cmd.changed + + - name: Wait 30s for Operator Deployment to terminate + k8s_info: + api_version: '{{ definition.apiVersion }}' + kind: '{{ definition.kind }}' + namespace: '{{ operator_namespace }}' + name: '{{ definition.metadata.name }}' + vars: + definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" + register: deployment + until: not deployment.resources + delay: 3 + retries: 10 + when: delete_deployment.changed + + - name: Create the Operator Deployment + k8s: + namespace: '{{ operator_namespace }}' + definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) }}" + + - name: Ensure the Tower custom_resource namespace exists + k8s: + state: present + name: '{{ custom_resource.metadata.namespace }}' + kind: Namespace + api_version: v1 + + - name: Create the tower.ansible.com/v1alpha1.Tower + k8s: + state: present + namespace: '{{ custom_resource.metadata.namespace }}' + definition: '{{ custom_resource }}' + + - name: Wait 5m for reconciliation to run + k8s_info: + api_version: '{{ custom_resource.apiVersion }}' + kind: '{{ custom_resource.kind }}' + namespace: '{{ custom_resource.metadata.namespace }}' + name: '{{ custom_resource.metadata.name }}' + register: cr + until: + - "'Successful' in (cr | json_query('resources[].status.conditions[].reason'))" + delay: 6 + retries: 50 + + rescue: + + - name: debug cr + ignore_errors: yes + failed_when: false + debug: + var: debug_cr + vars: + debug_cr: '{{ lookup("k8s", + kind=custom_resource.kind, + api_version=custom_resource.apiVersion, + namespace=custom_resource.metadata.namespace, + resource_name=custom_resource.metadata.name + )}}' + + - name: debug tower deployment + ignore_errors: yes + failed_when: false + debug: + var: deploy + vars: + deploy: '{{ lookup("k8s", + kind="Deployment", + api_version="apps/v1", + namespace=custom_resource.metadata.namespace, + label_selector="app=tower" + )}}' + + - name: get operator logs + ignore_errors: yes + failed_when: false + command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ operator_namespace }} -c operator + environment: + KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}' + vars: + definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" + register: log + + - debug: var=log.stdout_lines + + - fail: + msg: "Failed on action: converge" + +- import_playbook: '{{ playbook_dir }}/../default/asserts.yml' diff --git a/molecule/test-minikube/prepare.yml b/molecule/test-minikube/prepare.yml new file mode 100644 index 00000000..8ca82682 --- /dev/null +++ b/molecule/test-minikube/prepare.yml @@ -0,0 +1,2 @@ +--- +- import_playbook: ../default/prepare.yml