From 0b37f76225c07d815438bd880e095cd5f423e03e Mon Sep 17 00:00:00 2001 From: Hao Liu <44379968+TheRealHaoLiu@users.noreply.github.com> Date: Fri, 12 Apr 2024 13:52:48 -0400 Subject: [PATCH] Output debug resource to file in molecule test (#1823) - output all relevant k8s resource to file on failure - output awx job list and job details to file on failure - output all pod logs to file on failure - added STORE_DEBUG_OUTPUT to enable debug output gathering - added DEBUG_OUTPUT_DIR to control where the debug output files will be stored - when molecule test fail in CI trigger artifact gathering --- .github/workflows/ci.yaml | 9 + molecule/default/molecule.yml | 2 + molecule/default/tasks/apply_awx_spec.yml | 2 +- molecule/default/tasks/awx_test.yml | 292 +++++++++--------- .../output_all_container_logs_for_pod.yml | 15 + .../default/utils/output_k8s_resources.yml | 29 ++ molecule/default/verify.yml | 62 ++-- molecule/kind/molecule.yml | 2 + 8 files changed, 233 insertions(+), 180 deletions(-) create mode 100644 molecule/default/utils/output_all_container_logs_for_pod.yml create mode 100644 molecule/default/utils/output_k8s_resources.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41724d23..90f7cfd5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,6 +17,7 @@ jobs: - -t replicas env: DOCKER_API_VERSION: "1.41" + DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test steps: - uses: actions/checkout@v3 @@ -37,10 +38,18 @@ jobs: MOLECULE_VERBOSITY: 3 PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' + STORE_DEBUG_OUTPUT: true run: | sudo rm -f $(which kustomize) make kustomize KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }} + + - name: Upload artifacts for failed tests if Run Molecule fails + if: failure() + uses: actions/upload-artifact@v2 + with: + name: awx_operator_molecule_test + path: ${{ env.DEBUG_OUTPUT_DIR }} helm: runs-on: ubuntu-latest name: helm diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index f1f4120e..9a1898f2 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -31,6 +31,8 @@ provisioner: operator_image: ${OPERATOR_IMAGE:-""} operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"} kustomize: ${KUSTOMIZE_PATH:-kustomize} + store_debug_output: ${STORE_DEBUG_OUTPUT:-false} + debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"} env: K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"} verifier: diff --git a/molecule/default/tasks/apply_awx_spec.yml b/molecule/default/tasks/apply_awx_spec.yml index abd5708b..01aff07b 100644 --- a/molecule/default/tasks/apply_awx_spec.yml +++ b/molecule/default/tasks/apply_awx_spec.yml @@ -1,5 +1,5 @@ --- -- name: Create or update the awx.ansible.com/v1alpha1.AWX +- name: Create or update the awx.ansible.com/v1beta1.AWX k8s: state: present namespace: '{{ namespace }}' diff --git a/molecule/default/tasks/awx_test.yml b/molecule/default/tasks/awx_test.yml index f6f3757d..aaac0b82 100644 --- a/molecule/default/tasks/awx_test.yml +++ b/molecule/default/tasks/awx_test.yml @@ -1,98 +1,8 @@ --- - include_tasks: apply_awx_spec.yml -- name: Obtain generated admin password - k8s_info: - namespace: '{{ namespace }}' - kind: Secret - name: example-awx-admin-password - register: admin_pw_secret - -- block: - - name: Get web pod details - k8s_info: - namespace: '{{ namespace }}' - kind: Pod - label_selectors: - - app.kubernetes.io/name = example-awx-web - register: awx_web_pod - when: not awx_version - - - name: Get task pod details - k8s_info: - namespace: '{{ namespace }}' - kind: Pod - label_selectors: - - app.kubernetes.io/name = example-awx-task - register: awx_task_pod - when: not awx_version - - - name: Extract tags from images from web pod - set_fact: - web_image_tags: | - {{ awx_web_pod.resources[0].spec.containers | - map(attribute='image') | - map('regex_search', default_awx_version) }} - when: not awx_version - - - name: Extract tags from images from task pod - set_fact: - task_image_tags: | - {{ awx_task_pod.resources[0].spec.containers | - map(attribute='image') | - map('regex_search', default_awx_version) }} - when: not awx_version - - - fail: - msg: | - It looks like you may have broken the DEFAULT_AWX_VERSION functionality. - This is an environment variable that is set via build arg when releasing awx-operator. - when: - - not awx_version - - default_awx_version not in web_image_tags - - default_awx_version not in task_image_tags - - - name: Launch Demo Job Template - awx.awx.job_launch: - name: Demo Job Template - wait: yes - validate_certs: no - controller_host: localhost/awx/ - controller_username: admin - controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}" - rescue: - - name: Get list of project updates and jobs - uri: - url: "http://localhost/awx/api/v2/{{ resource }}/" - user: admin - password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}" - force_basic_auth: yes - register: job_lists - loop: - - project_updates - - jobs - loop_control: - loop_var: resource - - - name: Get all job and project details - uri: - url: "http://localhost{{ endpoint }}" - user: admin - password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}" - force_basic_auth: yes - loop: | - {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }} - loop_control: - loop_var: endpoint - - - name: Re-emit failure - vars: - failed_task: - result: '{{ ansible_failed_result }}' - fail: - msg: '{{ failed_task }}' - -- block: +- name: Validate AWX deployment + block: - name: Look up details for this AWX instance k8s_info: namespace: "{{ namespace }}" @@ -117,65 +27,90 @@ - app.kubernetes.io/name = example-awx-task register: awx_task_pod - - name: Extract additional_labels from AWX spec - set_fact: - awx_additional_labels: >- - {{ this_awx.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) - | list - }} + - name: Validate DEFAULT_AWX_VERSION + block: + - name: Extract tags from images from web pod + set_fact: + web_image_tags: | + {{ awx_web_pod.resources[0].spec.containers | + map(attribute='image') | + map('regex_search', default_awx_version) }} + - name: Extract tags from images from task pod + set_fact: + task_image_tags: | + {{ awx_task_pod.resources[0].spec.containers | + map(attribute='image') | + map('regex_search', default_awx_version) }} + - fail: + msg: | + It looks like you may have broken the DEFAULT_AWX_VERSION functionality. + This is an environment variable that is set via build arg when releasing awx-operator. + when: + - default_awx_version not in web_image_tags + - default_awx_version not in task_image_tags + when: not awx_version - - name: Extract additional_labels from AWX web Pod - set_fact: - awx_web_pod_additional_labels: >- - {{ awx_web_pod.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) - | list - }} - - - name: Extract additional_labels from AWX task Pod - set_fact: - awx_task_pod_additional_labels: >- - {{ awx_task_pod.resources[0].metadata.labels + - name: Validate additional_labels + block: + - name: Extract additional_labels from AWX spec + set_fact: + awx_additional_labels: >- + {{ this_awx.resources[0].metadata.labels | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) | list - }} + }} - - name: Assert AWX web Pod contains additional_labels - ansible.builtin.assert: - that: - - awx_web_pod_additional_labels == awx_additional_labels + - name: Extract additional_labels from AWX web Pod + set_fact: + awx_web_pod_additional_labels: >- + {{ awx_web_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) + | list + }} - - name: Assert AWX task Pod contains additional_labels - ansible.builtin.assert: - that: - - awx_task_pod_additional_labels == awx_additional_labels + - name: Extract additional_labels from AWX task Pod + set_fact: + awx_task_pod_additional_labels: >- + {{ awx_task_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) + | list + }} - - name: Extract web Pod labels which shouldn't have been propagated to it from AWX - set_fact: - awx_web_pod_extra_labels: >- - {{ awx_web_pod.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) - | list - }} + - name: Assert AWX web Pod contains additional_labels + ansible.builtin.assert: + that: + - awx_web_pod_additional_labels == awx_additional_labels - - name: AWX web Pod doesn't contain AWX labels not in additional_labels - ansible.builtin.assert: - that: - - awx_web_pod_extra_labels == [] + - name: Assert AWX task Pod contains additional_labels + ansible.builtin.assert: + that: + - awx_task_pod_additional_labels == awx_additional_labels - - name: Extract task Pod labels which shouldn't have been propagated to it from AWX - set_fact: - awx_task_pod_extra_labels: >- - {{ awx_task_pod.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) - | list - }} + - name: Extract web Pod labels which shouldn't have been propagated to it from AWX + set_fact: + awx_web_pod_extra_labels: >- + {{ awx_web_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) + | list + }} - - name: AWX task Pod doesn't contain AWX labels not in additional_labels - ansible.builtin.assert: - that: - - awx_task_pod_extra_labels == [] + - name: AWX web Pod doesn't contain AWX labels not in additional_labels + ansible.builtin.assert: + that: + - awx_web_pod_extra_labels == [] + + - name: Extract task Pod labels which shouldn't have been propagated to it from AWX + set_fact: + awx_task_pod_extra_labels: >- + {{ awx_task_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) + | list + }} + + - name: AWX task Pod doesn't contain AWX labels not in additional_labels + ansible.builtin.assert: + that: + - awx_task_pod_extra_labels == [] rescue: - name: Re-emit failure @@ -184,3 +119,76 @@ result: '{{ ansible_failed_result }}' fail: msg: '{{ failed_task }}' + +- name: Obtain generated admin password + k8s_info: + namespace: '{{ namespace }}' + kind: Secret + name: example-awx-admin-password + register: admin_pw_secret + +- name: Validate demo job launch + block: + - name: Launch Demo Job Template + awx.awx.job_launch: + name: Demo Job Template + wait: yes + validate_certs: no + controller_host: localhost/awx/ + controller_username: admin + controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}" + + rescue: + - name: Get list of project updates and jobs + uri: + url: "http://localhost/awx/api/v2/{{ resource }}/" + user: admin + password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}" + force_basic_auth: yes + register: job_lists + loop: + - project_updates + - jobs + loop_control: + loop_var: resource + + - name: Store job_lists debug output + copy: + content: "{{ job_lists | to_nice_json }}" + dest: "{{ debug_output_dir }}/job_lists.json" + when: store_debug_output | default(false) + + - name: Get all job and project details + uri: + url: "http://localhost{{ endpoint }}" + user: admin + password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}" + force_basic_auth: yes + loop: | + {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }} + loop_control: + loop_var: endpoint + register: job_details + + - name: Store job_details debug output + copy: + content: "{{ job_details | to_nice_json }}" + dest: "{{ debug_output_dir }}/job_details.json" + when: store_debug_output | default(false) + + ## TODO: figure out why this doesn't work + # - name: Store debug outputs + # copy: + # content: '{{ item }}' + # dest: "{{ debug_output_dir }}/{{ item }}.json" + # loop: + # - job_lists + # - job_details + # when: store_debug_output | default(false) + + - name: Re-emit failure + vars: + failed_task: + result: '{{ ansible_failed_result }}' + fail: + msg: '{{ failed_task }}' diff --git a/molecule/default/utils/output_all_container_logs_for_pod.yml b/molecule/default/utils/output_all_container_logs_for_pod.yml new file mode 100644 index 00000000..d4e32461 --- /dev/null +++ b/molecule/default/utils/output_all_container_logs_for_pod.yml @@ -0,0 +1,15 @@ +--- +- name: Get all container log in pod + kubernetes.core.k8s_log: + namespace: '{{ namespace }}' + name: '{{ item.metadata.name }}' + all_containers: true + register: all_container_logs + +- name: Store logs in file + ansible.builtin.copy: + content: "{{ all_container_logs.log_lines | join('\n') }}" + dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log' + +# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs +# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported diff --git a/molecule/default/utils/output_k8s_resources.yml b/molecule/default/utils/output_k8s_resources.yml new file mode 100644 index 00000000..71fb1374 --- /dev/null +++ b/molecule/default/utils/output_k8s_resources.yml @@ -0,0 +1,29 @@ +--- +- name: Retrieve relevant k8s resources + kubernetes.core.k8s_info: + api_version: '{{ item.api_version }}' + kind: '{{ item.kind }}' + namespace: '{{ namespace }}' + loop: + - api_version: v1 + kind: Pod + - api_version: apps/v1 + kind: Deployment + - api_version: v1 + kind: Secret + - api_version: v1 + kind: ConfigMap + - api_version: "awx.ansible.com/v1beta1" + kind: AWX + register: debug_resources + +- name: debug print item.kind and item.metadata.name + debug: + msg: '{{ item.kind }}-{{ item.metadata.name }}' + loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}" + +- name: Output gathered resource to files + ansible.builtin.copy: + content: '{{ item | to_nice_json }}' + dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json' + loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}" diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index 78733c04..31b95d3f 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -10,53 +10,41 @@ ctrl_label: control-plane=controller-manager tasks: - - block: + - name: Perform awx tests + block: - name: Import all test files from tasks/ - include_tasks: '{{ item }}' + ansible.builtin.include_tasks: '{{ item }}' with_fileglob: - tasks/awx_test.yml - tasks/awx_replicas_test.yml tags: - always rescue: - - name: Retrieve relevant resources - k8s_info: - api_version: '{{ item.api_version }}' - kind: '{{ item.kind }}' + - name: Create debug output directory + ansible.builtin.file: + path: '{{ debug_output_dir }}' + state: directory + tags: + - always + + - name: Gather and output K8s resources + ansible.builtin.include_tasks: utils/output_k8s_resources.yml + tags: + - always + + - name: Get all pods + kubernetes.core.k8s_info: + api_version: v1 + kind: Pod namespace: '{{ namespace }}' - loop: - - api_version: v1 - kind: Pod - - api_version: apps/v1 - kind: Deployment - - api_version: v1 - kind: Secret - - api_version: v1 - kind: ConfigMap - register: debug_resources + register: all_pods tags: - always - - name: Retrieve Pod logs - k8s_log: - name: '{{ item.metadata.name }}' - namespace: '{{ namespace }}' - container: awx-manager - loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}" - register: debug_logs - tags: - - always - - - name: Output gathered resources - debug: - var: debug_resources - tags: - - always - - - name: Output gathered logs - debug: - var: item.log_lines - loop: '{{ debug_logs.results }}' + - name: Get all container logs for all pods + ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml + loop: '{{ all_pods.resources }}' + ignore_errors: yes tags: - always @@ -64,7 +52,7 @@ vars: failed_task: result: '{{ ansible_failed_result }}' - fail: + ansible.builtin.fail: msg: '{{ failed_task }}' tags: - always diff --git a/molecule/kind/molecule.yml b/molecule/kind/molecule.yml index 54658123..e3823123 100644 --- a/molecule/kind/molecule.yml +++ b/molecule/kind/molecule.yml @@ -35,6 +35,8 @@ provisioner: operator_pull_policy: "Never" kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" kustomize: ${KUSTOMIZE_PATH:-kustomize} + store_debug_output: ${STORE_DEBUG_OUTPUT:-false} + debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"} env: K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig