internet/awx-operator
3
0
Fork 0
mirror of https://github.com/ansible/awx-operator.git synced 2026-03-26 21:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Output debug resource to file in molecule test (#1823)

Browse Source 
- output all relevant k8s resource to file on failure
- output awx job list and job details to file on failure
- output all pod logs to file on failure
- added STORE_DEBUG_OUTPUT to enable debug output gathering
- added DEBUG_OUTPUT_DIR to control where the debug output files will be stored
- when molecule test fail in CI trigger artifact gathering
This commit is contained in:
Hao Liu
2024-04-12 13:52:48 -04:00
committed by GitHub
parent e6e1025206
commit 0b37f76225
8 changed files with 233 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/ci.yaml vendored
View File

@@ -17,6 +17,7 @@ jobs:
- -t replicas
env:
DOCKER_API_VERSION: "1.41"
DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
steps:
- uses: actions/checkout@v3
@@ -37,10 +38,18 @@ jobs:
MOLECULE_VERBOSITY: 3
PY_COLORS: '1'
ANSIBLE_FORCE_COLOR: '1'
STORE_DEBUG_OUTPUT: true
run: |
sudo rm -f $(which kustomize)
make kustomize
KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }}
- name: Upload artifacts for failed tests if Run Molecule fails
if: failure()
uses: actions/upload-artifact@v2
with:
name: awx_operator_molecule_test
path: ${{ env.DEBUG_OUTPUT_DIR }}
helm:
runs-on: ubuntu-latest
name: helm

2
molecule/default/molecule.yml
View File

@@ -31,6 +31,8 @@ provisioner:
operator_image: ${OPERATOR_IMAGE:-""}
operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"}
kustomize: ${KUSTOMIZE_PATH:-kustomize}
store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
env:
K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"}
verifier:

2
molecule/default/tasks/apply_awx_spec.yml
View File

@@ -1,5 +1,5 @@
---
- name: Create or update the awx.ansible.com/v1alpha1.AWX
- name: Create or update the awx.ansible.com/v1beta1.AWX
k8s:
state: present
namespace: '{{ namespace }}'

292
molecule/default/tasks/awx_test.yml
View File

@@ -1,98 +1,8 @@
---
- include_tasks: apply_awx_spec.yml
- name: Obtain generated admin password
k8s_info:
namespace: '{{ namespace }}'
kind: Secret
name: example-awx-admin-password
register: admin_pw_secret
- block:
- name: Get web pod details
k8s_info:
namespace: '{{ namespace }}'
kind: Pod
label_selectors:
- app.kubernetes.io/name = example-awx-web
register: awx_web_pod
when: not awx_version
- name: Get task pod details
k8s_info:
namespace: '{{ namespace }}'
kind: Pod
label_selectors:
- app.kubernetes.io/name = example-awx-task
register: awx_task_pod
when: not awx_version
- name: Extract tags from images from web pod
set_fact:
web_image_tags: |
{{ awx_web_pod.resources[0].spec.containers |
map(attribute='image') |
map('regex_search', default_awx_version) }}
when: not awx_version
- name: Extract tags from images from task pod
set_fact:
task_image_tags: |
{{ awx_task_pod.resources[0].spec.containers |
map(attribute='image') |
map('regex_search', default_awx_version) }}
when: not awx_version
- fail:
msg: |
It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
This is an environment variable that is set via build arg when releasing awx-operator.
when:
- not awx_version
- default_awx_version not in web_image_tags
- default_awx_version not in task_image_tags
- name: Launch Demo Job Template
awx.awx.job_launch:
name: Demo Job Template
wait: yes
validate_certs: no
controller_host: localhost/awx/
controller_username: admin
controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
rescue:
- name: Get list of project updates and jobs
uri:
url: "http://localhost/awx/api/v2/{{ resource }}/"
user: admin
password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
force_basic_auth: yes
register: job_lists
loop:
- project_updates
- jobs
loop_control:
loop_var: resource
- name: Get all job and project details
uri:
url: "http://localhost{{ endpoint }}"
user: admin
password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
force_basic_auth: yes
loop: |
{{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
loop_control:
loop_var: endpoint
- name: Re-emit failure
vars:
failed_task:
result: '{{ ansible_failed_result }}'
fail:
msg: '{{ failed_task }}'
- block:
- name: Validate AWX deployment
block:
- name: Look up details for this AWX instance
k8s_info:
namespace: "{{ namespace }}"
@@ -117,65 +27,90 @@
- app.kubernetes.io/name = example-awx-task
register: awx_task_pod
- name: Extract additional_labels from AWX spec
set_fact:
awx_additional_labels: >-
{{ this_awx.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
| list
}}
- name: Validate DEFAULT_AWX_VERSION
block:
- name: Extract tags from images from web pod
set_fact:
web_image_tags: |
{{ awx_web_pod.resources[0].spec.containers |
map(attribute='image') |
map('regex_search', default_awx_version) }}
- name: Extract tags from images from task pod
set_fact:
task_image_tags: |
{{ awx_task_pod.resources[0].spec.containers |
map(attribute='image') |
map('regex_search', default_awx_version) }}
- fail:
msg: |
It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
This is an environment variable that is set via build arg when releasing awx-operator.
when:
- default_awx_version not in web_image_tags
- default_awx_version not in task_image_tags
when: not awx_version
- name: Extract additional_labels from AWX web Pod
set_fact:
awx_web_pod_additional_labels: >-
{{ awx_web_pod.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
| list
}}
- name: Extract additional_labels from AWX task Pod
set_fact:
awx_task_pod_additional_labels: >-
{{ awx_task_pod.resources[0].metadata.labels
- name: Validate additional_labels
block:
- name: Extract additional_labels from AWX spec
set_fact:
awx_additional_labels: >-
{{ this_awx.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
| list
}}
}}
- name: Assert AWX web Pod contains additional_labels
ansible.builtin.assert:
that:
- awx_web_pod_additional_labels == awx_additional_labels
- name: Extract additional_labels from AWX web Pod
set_fact:
awx_web_pod_additional_labels: >-
{{ awx_web_pod.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
| list
}}
- name: Assert AWX task Pod contains additional_labels
ansible.builtin.assert:
that:
- awx_task_pod_additional_labels == awx_additional_labels
- name: Extract additional_labels from AWX task Pod
set_fact:
awx_task_pod_additional_labels: >-
{{ awx_task_pod.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
| list
}}
- name: Extract web Pod labels which shouldn't have been propagated to it from AWX
set_fact:
awx_web_pod_extra_labels: >-
{{ awx_web_pod.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
| list
}}
- name: Assert AWX web Pod contains additional_labels
ansible.builtin.assert:
that:
- awx_web_pod_additional_labels == awx_additional_labels
- name: AWX web Pod doesn't contain AWX labels not in additional_labels
ansible.builtin.assert:
that:
- awx_web_pod_extra_labels == []
- name: Assert AWX task Pod contains additional_labels
ansible.builtin.assert:
that:
- awx_task_pod_additional_labels == awx_additional_labels
- name: Extract task Pod labels which shouldn't have been propagated to it from AWX
set_fact:
awx_task_pod_extra_labels: >-
{{ awx_task_pod.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
| list
}}
- name: Extract web Pod labels which shouldn't have been propagated to it from AWX
set_fact:
awx_web_pod_extra_labels: >-
{{ awx_web_pod.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
| list
}}
- name: AWX task Pod doesn't contain AWX labels not in additional_labels
ansible.builtin.assert:
that:
- awx_task_pod_extra_labels == []
- name: AWX web Pod doesn't contain AWX labels not in additional_labels
ansible.builtin.assert:
that:
- awx_web_pod_extra_labels == []
- name: Extract task Pod labels which shouldn't have been propagated to it from AWX
set_fact:
awx_task_pod_extra_labels: >-
{{ awx_task_pod.resources[0].metadata.labels
| dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
| list
}}
- name: AWX task Pod doesn't contain AWX labels not in additional_labels
ansible.builtin.assert:
that:
- awx_task_pod_extra_labels == []
rescue:
- name: Re-emit failure
@@ -184,3 +119,76 @@
result: '{{ ansible_failed_result }}'
fail:
msg: '{{ failed_task }}'
- name: Obtain generated admin password
k8s_info:
namespace: '{{ namespace }}'
kind: Secret
name: example-awx-admin-password
register: admin_pw_secret
- name: Validate demo job launch
block:
- name: Launch Demo Job Template
awx.awx.job_launch:
name: Demo Job Template
wait: yes
validate_certs: no
controller_host: localhost/awx/
controller_username: admin
controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
rescue:
- name: Get list of project updates and jobs
uri:
url: "http://localhost/awx/api/v2/{{ resource }}/"
user: admin
password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
force_basic_auth: yes
register: job_lists
loop:
- project_updates
- jobs
loop_control:
loop_var: resource
- name: Store job_lists debug output
copy:
content: "{{ job_lists | to_nice_json }}"
dest: "{{ debug_output_dir }}/job_lists.json"
when: store_debug_output | default(false)
- name: Get all job and project details
uri:
url: "http://localhost{{ endpoint }}"
user: admin
password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
force_basic_auth: yes
loop: |
{{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
loop_control:
loop_var: endpoint
register: job_details
- name: Store job_details debug output
copy:
content: "{{ job_details | to_nice_json }}"
dest: "{{ debug_output_dir }}/job_details.json"
when: store_debug_output | default(false)
## TODO: figure out why this doesn't work
# - name: Store debug outputs
# copy:
# content: '{{ item }}'
# dest: "{{ debug_output_dir }}/{{ item }}.json"
# loop:
# - job_lists
# - job_details
# when: store_debug_output | default(false)
- name: Re-emit failure
vars:
failed_task:
result: '{{ ansible_failed_result }}'
fail:
msg: '{{ failed_task }}'

15
molecule/default/utils/output_all_container_logs_for_pod.yml Normal file
View File

@@ -0,0 +1,15 @@
---
- name: Get all container log in pod
kubernetes.core.k8s_log:
namespace: '{{ namespace }}'
name: '{{ item.metadata.name }}'
all_containers: true
register: all_container_logs
- name: Store logs in file
ansible.builtin.copy:
content: "{{ all_container_logs.log_lines | join('\n') }}"
dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log'
# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs
# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported

29
molecule/default/utils/output_k8s_resources.yml Normal file
View File

@@ -0,0 +1,29 @@
---
- name: Retrieve relevant k8s resources
kubernetes.core.k8s_info:
api_version: '{{ item.api_version }}'
kind: '{{ item.kind }}'
namespace: '{{ namespace }}'
loop:
- api_version: v1
kind: Pod
- api_version: apps/v1
kind: Deployment
- api_version: v1
kind: Secret
- api_version: v1
kind: ConfigMap
- api_version: "awx.ansible.com/v1beta1"
kind: AWX
register: debug_resources
- name: debug print item.kind and item.metadata.name
debug:
msg: '{{ item.kind }}-{{ item.metadata.name }}'
loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"
- name: Output gathered resource to files
ansible.builtin.copy:
content: '{{ item | to_nice_json }}'
dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json'
loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"

62
molecule/default/verify.yml
View File

@@ -10,53 +10,41 @@
ctrl_label: control-plane=controller-manager
tasks:
- block:
- name: Perform awx tests
block:
- name: Import all test files from tasks/
include_tasks: '{{ item }}'
ansible.builtin.include_tasks: '{{ item }}'
with_fileglob:
- tasks/awx_test.yml
- tasks/awx_replicas_test.yml
tags:
- always
rescue:
- name: Retrieve relevant resources
k8s_info:
api_version: '{{ item.api_version }}'
kind: '{{ item.kind }}'
- name: Create debug output directory
ansible.builtin.file:
path: '{{ debug_output_dir }}'
state: directory
tags:
- always
- name: Gather and output K8s resources
ansible.builtin.include_tasks: utils/output_k8s_resources.yml
tags:
- always
- name: Get all pods
kubernetes.core.k8s_info:
api_version: v1
kind: Pod
namespace: '{{ namespace }}'
loop:
- api_version: v1
kind: Pod
- api_version: apps/v1
kind: Deployment
- api_version: v1
kind: Secret
- api_version: v1
kind: ConfigMap
register: debug_resources
register: all_pods
tags:
- always
- name: Retrieve Pod logs
k8s_log:
name: '{{ item.metadata.name }}'
namespace: '{{ namespace }}'
container: awx-manager
loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}"
register: debug_logs
tags:
- always
- name: Output gathered resources
debug:
var: debug_resources
tags:
- always
- name: Output gathered logs
debug:
var: item.log_lines
loop: '{{ debug_logs.results }}'
- name: Get all container logs for all pods
ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml
loop: '{{ all_pods.resources }}'
ignore_errors: yes
tags:
- always
@@ -64,7 +52,7 @@
vars:
failed_task:
result: '{{ ansible_failed_result }}'
fail:
ansible.builtin.fail:
msg: '{{ failed_task }}'
tags:
- always

2
molecule/kind/molecule.yml
View File

@@ -35,6 +35,8 @@ provisioner:
operator_pull_policy: "Never"
kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
kustomize: ${KUSTOMIZE_PATH:-kustomize}
store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
env:
K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig
KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig