Output debug resource to file in molecule test (#1823)

- output all relevant k8s resource to file on failure
- output awx job list and job details to file on failure
- output all pod logs to file on failure
- added STORE_DEBUG_OUTPUT to enable debug output gathering
- added DEBUG_OUTPUT_DIR to control where the debug output files will be stored
- when molecule test fail in CI trigger artifact gathering

.github/workflows/ci.yaml

@@ -17,6 +17,7 @@ jobs:
           - -t replicas
     env:
       DOCKER_API_VERSION: "1.41"
+      DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
     steps:
       - uses: actions/checkout@v3
 
@@ -37,10 +38,18 @@ jobs:
           MOLECULE_VERBOSITY: 3
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          STORE_DEBUG_OUTPUT: true
         run: |
           sudo rm -f $(which kustomize)
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }}
+
+      - name: Upload artifacts for failed tests if Run Molecule fails
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: awx_operator_molecule_test
+          path: ${{ env.DEBUG_OUTPUT_DIR }}
   helm:
     runs-on: ubuntu-latest
     name: helm

molecule/default/molecule.yml

@@ -31,6 +31,8 @@ provisioner:
         operator_image: ${OPERATOR_IMAGE:-""}
         operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"}
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"}
 verifier:

molecule/default/tasks/apply_awx_spec.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create or update the awx.ansible.com/v1alpha1.AWX
+- name: Create or update the awx.ansible.com/v1beta1.AWX
   k8s:
     state: present
     namespace: '{{ namespace }}'

molecule/default/tasks/awx_test.yml

@@ -1,98 +1,8 @@
 ---
 - include_tasks: apply_awx_spec.yml
 
-- name: Obtain generated admin password
+- name: Validate AWX deployment
-  k8s_info:
+  block:
-    namespace: '{{ namespace }}'
-    kind: Secret
-    name: example-awx-admin-password
-  register: admin_pw_secret
-
-- block:
-    - name: Get web pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-web
-      register: awx_web_pod
-      when: not awx_version
-
-    - name: Get task pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-task
-      register: awx_task_pod
-      when: not awx_version
-
-    - name: Extract tags from images from web pod
-      set_fact:
-        web_image_tags: |
-          {{ awx_web_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - name: Extract tags from images from task pod
-      set_fact:
-        task_image_tags: |
-          {{ awx_task_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - fail:
-        msg: |
-          It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
-          This is an environment variable that is set via build arg when releasing awx-operator.
-      when:
-        - not awx_version
-        - default_awx_version not in web_image_tags
-        - default_awx_version not in task_image_tags
-
-    - name: Launch Demo Job Template
-      awx.awx.job_launch:
-        name: Demo Job Template
-        wait: yes
-        validate_certs: no
-        controller_host: localhost/awx/
-        controller_username: admin
-        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-  rescue:
-    - name: Get list of project updates and jobs
-      uri:
-        url: "http://localhost/awx/api/v2/{{ resource }}/"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      register: job_lists
-      loop:
-        - project_updates
-        - jobs
-      loop_control:
-        loop_var: resource
-
-    - name: Get all job and project details
-      uri:
-        url: "http://localhost{{ endpoint }}"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      loop: |
-        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
-      loop_control:
-        loop_var: endpoint
-
-    - name: Re-emit failure
-      vars:
-        failed_task:
-          result: '{{ ansible_failed_result }}'
-      fail:
-        msg: '{{ failed_task }}'
-
-- block:
     - name: Look up details for this AWX instance
       k8s_info:
         namespace: "{{ namespace }}"
@@ -117,65 +27,90 @@
           - app.kubernetes.io/name = example-awx-task
       register: awx_task_pod
 
-    - name: Extract additional_labels from AWX spec
+    - name: Validate DEFAULT_AWX_VERSION
-      set_fact:
+      block:
-        awx_additional_labels: >-
+      - name: Extract tags from images from web pod
-          {{ this_awx.resources[0].metadata.labels
+        set_fact:
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+          web_image_tags: |
-             | list
+            {{ awx_web_pod.resources[0].spec.containers |
-          }}
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - name: Extract tags from images from task pod
+        set_fact:
+          task_image_tags: |
+            {{ awx_task_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - fail:
+          msg: |
+            It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
+            This is an environment variable that is set via build arg when releasing awx-operator.
+        when:
+          - default_awx_version not in web_image_tags
+          - default_awx_version not in task_image_tags
+      when: not awx_version
 
-    - name: Extract additional_labels from AWX web Pod
+    - name: Validate additional_labels
-      set_fact:
+      block:
-        awx_web_pod_additional_labels: >-
+      - name: Extract additional_labels from AWX spec
-          {{ awx_web_pod.resources[0].metadata.labels
+        set_fact:
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+          awx_additional_labels: >-
-             | list
+            {{ this_awx.resources[0].metadata.labels
-          }}
-
-    - name: Extract additional_labels from AWX task Pod
-      set_fact:
-        awx_task_pod_additional_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
               | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
               | list
-          }}
+            }}
 
-    - name: Assert AWX web Pod contains additional_labels
+      - name: Extract additional_labels from AWX web Pod
-      ansible.builtin.assert:
+        set_fact:
-        that:
+          awx_web_pod_additional_labels: >-
-          - awx_web_pod_additional_labels == awx_additional_labels
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+              | list
+            }}
 
-    - name: Assert AWX task Pod contains additional_labels
+      - name: Extract additional_labels from AWX task Pod
-      ansible.builtin.assert:
+        set_fact:
-        that:
+          awx_task_pod_additional_labels: >-
-          - awx_task_pod_additional_labels == awx_additional_labels
+            {{ awx_task_pod.resources[0].metadata.labels
+                | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+                | list
+            }}
 
-    - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
+      - name: Assert AWX web Pod contains additional_labels
-      set_fact:
+        ansible.builtin.assert:
-        awx_web_pod_extra_labels: >-
+          that:
-          {{ awx_web_pod.resources[0].metadata.labels
+            - awx_web_pod_additional_labels == awx_additional_labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
 
-    - name: AWX web Pod doesn't contain AWX labels not in additional_labels
+      - name: Assert AWX task Pod contains additional_labels
-      ansible.builtin.assert:
+        ansible.builtin.assert:
-        that:
+          that:
-          - awx_web_pod_extra_labels == []
+            - awx_task_pod_additional_labels == awx_additional_labels
 
-    - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
+      - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
+        set_fact:
-        awx_task_pod_extra_labels: >-
+          awx_web_pod_extra_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
+            {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
+              | list
-          }}
+            }}
 
-    - name: AWX task Pod doesn't contain AWX labels not in additional_labels
+      - name: AWX web Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
+        ansible.builtin.assert:
-        that:
+          that:
-          - awx_task_pod_extra_labels == []
+            - awx_web_pod_extra_labels == []
+
+      - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_task_pod_extra_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
+
+      - name: AWX task Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_extra_labels == []
 
   rescue:
     - name: Re-emit failure
@@ -184,3 +119,76 @@
           result: '{{ ansible_failed_result }}'
       fail:
         msg: '{{ failed_task }}'
+
+- name: Obtain generated admin password
+  k8s_info:
+    namespace: '{{ namespace }}'
+    kind: Secret
+    name: example-awx-admin-password
+  register: admin_pw_secret
+
+- name: Validate demo job launch
+  block:
+    - name: Launch Demo Job Template
+      awx.awx.job_launch:
+        name: Demo Job Template
+        wait: yes
+        validate_certs: no
+        controller_host: localhost/awx/
+        controller_username: admin
+        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+
+  rescue:
+    - name: Get list of project updates and jobs
+      uri:
+        url: "http://localhost/awx/api/v2/{{ resource }}/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: job_lists
+      loop:
+        - project_updates
+        - jobs
+      loop_control:
+        loop_var: resource
+
+    - name: Store job_lists debug output
+      copy:
+        content: "{{ job_lists | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_lists.json"
+      when: store_debug_output | default(false)
+
+    - name: Get all job and project details
+      uri:
+        url: "http://localhost{{ endpoint }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
+      loop_control:
+        loop_var: endpoint
+      register: job_details
+
+    - name: Store job_details debug output
+      copy:
+        content: "{{ job_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_details.json"
+      when: store_debug_output | default(false)
+
+    ## TODO: figure out why this doesn't work
+    # - name: Store debug outputs
+    #   copy:
+    #     content: '{{ item }}'
+    #     dest: "{{ debug_output_dir }}/{{ item }}.json"
+    #   loop:
+    #     - job_lists
+    #     - job_details
+    #   when: store_debug_output | default(false)
+
+    - name: Re-emit failure
+      vars:
+        failed_task:
+          result: '{{ ansible_failed_result }}'
+      fail:
+        msg: '{{ failed_task }}'

molecule/default/utils/output_all_container_logs_for_pod.yml

@@ -0,0 +1,15 @@
+---
+- name: Get all container log in pod
+  kubernetes.core.k8s_log:
+    namespace: '{{ namespace }}'
+    name: '{{ item.metadata.name }}'
+    all_containers: true
+  register: all_container_logs
+
+- name: Store logs in file
+  ansible.builtin.copy:
+    content: "{{ all_container_logs.log_lines | join('\n') }}"
+    dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log'
+
+# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs
+# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported

molecule/default/utils/output_k8s_resources.yml

@@ -0,0 +1,29 @@
+---
+- name: Retrieve relevant k8s resources
+  kubernetes.core.k8s_info:
+    api_version: '{{ item.api_version }}'
+    kind: '{{ item.kind }}'
+    namespace: '{{ namespace }}'
+  loop:
+    - api_version: v1
+      kind: Pod
+    - api_version: apps/v1
+      kind: Deployment
+    - api_version: v1
+      kind: Secret
+    - api_version: v1
+      kind: ConfigMap
+    - api_version: "awx.ansible.com/v1beta1"
+      kind: AWX
+  register: debug_resources
+
+- name: debug print item.kind and item.metadata.name
+  debug:
+    msg: '{{ item.kind }}-{{ item.metadata.name }}'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"
+
+- name: Output gathered resource to files
+  ansible.builtin.copy:
+    content: '{{ item | to_nice_json }}'
+    dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"

molecule/default/verify.yml

@@ -10,53 +10,41 @@
     ctrl_label: control-plane=controller-manager
 
   tasks:
-    - block:
+    - name: Perform awx tests
+      block:
         - name: Import all test files from tasks/
-          include_tasks: '{{ item }}'
+          ansible.builtin.include_tasks: '{{ item }}'
           with_fileglob:
             - tasks/awx_test.yml
             - tasks/awx_replicas_test.yml
           tags:
             - always
       rescue:
-        - name: Retrieve relevant resources
+        - name: Create debug output directory
-          k8s_info:
+          ansible.builtin.file:
-            api_version: '{{ item.api_version }}'
+            path: '{{ debug_output_dir }}'
-            kind: '{{ item.kind }}'
+            state: directory
+          tags:
+            - always
+
+        - name: Gather and output K8s resources
+          ansible.builtin.include_tasks: utils/output_k8s_resources.yml
+          tags:
+            - always
+
+        - name: Get all pods
+          kubernetes.core.k8s_info:
+            api_version: v1
+            kind: Pod
             namespace: '{{ namespace }}'
-          loop:
+          register: all_pods
-            - api_version: v1
-              kind: Pod
-            - api_version: apps/v1
-              kind: Deployment
-            - api_version: v1
-              kind: Secret
-            - api_version: v1
-              kind: ConfigMap
-          register: debug_resources
           tags:
             - always
 
-        - name: Retrieve Pod logs
+        - name: Get all container logs for all pods
-          k8s_log:
+          ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml
-            name: '{{ item.metadata.name }}'
+          loop: '{{ all_pods.resources }}'
-            namespace: '{{ namespace }}'
+          ignore_errors: yes
-            container: awx-manager
-          loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}"
-          register: debug_logs
-          tags:
-            - always
-
-        - name: Output gathered resources
-          debug:
-            var: debug_resources
-          tags:
-            - always
-
-        - name: Output gathered logs
-          debug:
-            var: item.log_lines
-          loop: '{{ debug_logs.results }}'
           tags:
             - always
 
@@ -64,7 +52,7 @@
           vars:
             failed_task:
               result: '{{ ansible_failed_result }}'
-          fail:
+          ansible.builtin.fail:
             msg: '{{ failed_task }}'
           tags:
             - always

molecule/kind/molecule.yml

@@ -35,6 +35,8 @@ provisioner:
         operator_pull_policy: "Never"
         kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig
     KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig