Improve logging in CI (#1868)

- Set AWX log level to DEBUG
- Fix failure to collect awx API output

.helm/starter/templates/postgres-config.yaml

@@ -12,6 +12,7 @@ stringData:
   username: {{ .username }}
   password: {{ .password }}
   sslmode: {{ .sslmode }}
+  target_session_attrs: {{ .target_session_attrs | default "any" }}
   type: {{ .type }}
 type: Opaque
 {{- end }}

config/crd/bases/awx.ansible.com_awxrestores.yaml

@@ -94,6 +94,11 @@ spec:
               postgres_image_version:
                 description: PostgreSQL container image version to use
                 type: string
+              spec_overrides:
+                description: Overrides for the AWX spec
+                # type: string
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
               image_pull_policy:
                 description: The image pull policy
                 type: string

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -144,6 +144,9 @@ spec:
               ingress_controller:
                 description: Special configuration for specific Ingress Controllers
                 type: string
+              api_urlpattern_prefix:
+                description: An optional configuration to add a prefix in the API URL path
+                type: string
               loadbalancer_protocol:
                 description: Protocol to use for the loadbalancer
                 type: string

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -58,6 +58,11 @@ spec:
         path: ingress_controller
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Optional API URLPATTERN Prefix
+        path: api_urlpattern_prefix
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
       - displayName: Image Pull Secrets
         path: image_pull_secrets
         x-descriptors:
@@ -213,6 +218,10 @@ spec:
         path: postgres_image_version
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: AWX Spec Overrides
+        path: spec_overrides
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Image Pull Policy
         path: image_pull_policy
         x-descriptors:

docs/requirements.txt

@@ -94,7 +94,7 @@ mkdocs==1.5.3
     #   mkdocs-minify-plugin
     #   mkdocs-monorepo-plugin
     #   mkdocstrings
-mkdocs-ansible==24.3.0
+mkdocs-ansible==24.3.1
     # via -r requirements.in
 mkdocs-autorefs==0.5.0
     # via mkdocstrings

docs/user-guide/advanced-configuration/assigning-awx-pods-to-specific-nodes.md

@@ -88,3 +88,8 @@ spec:
               - S2
           topologyKey: topology.kubernetes.io/zone
 ```
+
+#### Special Note on DB-Migration Job Scheduling
+
+For the **db-migration job**, which applies database migrations at cluster startup, you can specify scheduling settings using the `task_*` configurations such as `task_node_selector`, `task_tolerations`, etc.  
+If these task-specific settings are not defined, the job will automatically use the global AWX configurations like `node_selector` and `tolerations`.

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -137,14 +137,57 @@ configuration.
 * [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size)
 
 
-##### Custom Favicon
+##### Custom Logos
 
-You can use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
+You can use custom volume mounts to mount in your own logos to be displayed instead of the AWX logo.
+There are two different logos, one to be displayed on page headers, and one for the login screen.
 
-First, Create the configmap from a local favicon.ico file.
+First, create configmaps for the logos from local `logo-login.svg` and `logo-header.svg` files.
 
 ```bash
-$ oc create configmap favicon-configmap --from-file favicon.ico
+$ kubectl create configmap logo-login-configmap --from-file logo-login.svg
+$ kubectl create configmap logo-header-configmap --from-file logo-header.svg
+```
+
+Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
+
+```yaml
+---
+spec:
+  ...
+  extra_volumes: |
+    - name: logo-login
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-login.svg
+            path: logo-login.svg
+        name: logo-login-configmap
+    - name: logo-header
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-header.svg
+            path: logo-header.svg
+        name: logo-header-configmap
+  web_extra_volume_mounts: |
+    - name: logo-login
+      mountPath: /var/lib/awx/public/static/media/logo-login.svg
+      subPath: logo-login.svg
+    - name: logo-header
+      mountPath: /var/lib/awx/public/static/media/logo-header.svg
+      subPath: logo-header.svg
+```
+
+
+##### Custom Favicon
+
+You can also use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
+
+First, create the configmap from a local `favicon.ico` file.
+
+```bash
+$ kubectl create configmap favicon-configmap --from-file favicon.ico
 ```
 
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec

docs/user-guide/database-configuration.md

@@ -27,6 +27,7 @@ stringData:
   username: <username to connect as>
   password: <password to connect with>
   sslmode: prefer
+  target_session_attrs: read-write
   type: unmanaged
 type: Opaque
 ```
@@ -37,6 +38,8 @@ type: Opaque
 
 **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`.
 
+**Note**: The variable `target_session_attrs` is only useful for `clustered external` databases. The allowed values are: `any` (default), `read-write`, `read-only`, `primary`, `standby` and `prefer-standby`, whereby only `read-write` and `primary` really make sense in AWX use, as you want to connect to a database node that offers write support.
+
 Once the secret is created, you can specify it on your spec:
 
 ```yaml

molecule/default/tasks/awx_test.yml

@@ -139,6 +139,11 @@
         controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
 
   rescue:
+    - name: Create debug output directory
+      ansible.builtin.file:
+        path: '{{ debug_output_dir }}'
+        state: directory
+
     - name: Get list of project updates and jobs
       uri:
         url: "http://localhost/awx/api/v2/{{ resource }}/"
@@ -158,7 +163,7 @@
         dest: "{{ debug_output_dir }}/job_lists.json"
       when: store_debug_output | default(false)
 
-    - name: Get all job and project details
+    - name: Get all job and project_update details
       uri:
         url: "http://localhost{{ endpoint }}"
         user: admin
@@ -176,6 +181,38 @@
         dest: "{{ debug_output_dir }}/job_details.json"
       when: store_debug_output | default(false)
 
+    - name: Get list of instances
+      uri:
+        url: "http://localhost/awx/api/v2/instances/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: instances_list
+
+    - name: Store instances_list debug output
+      copy:
+        content: "{{ instances_list | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_list.json"
+      when: store_debug_output | default(false)
+
+    - name: Get instances detail
+      uri:
+        url: "http://localhost{{ item }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ instances_list.json.results | map(attribute='url') }}
+      loop_control:
+        loop_var: item
+      register: instances_details
+
+    - name: Store instances_details debug output
+      copy:
+        content: "{{ instances_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_details.json"
+      when: store_debug_output | default(false)
+
     ## TODO: figure out why this doesn't work
     # - name: Store debug outputs
     #   copy:

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -42,6 +42,9 @@ spec:
   additional_labels:
   - my/team
   - my/service
+  extra_settings:
+  - setting: LOG_AGGREGATOR_LEVEL
+    value: "'DEBUG'"
 {% if additional_fields is defined %}
   {{ additional_fields | to_nice_yaml | indent(2) }}
 {% endif %}

molecule/requirements.yml

@@ -2,8 +2,6 @@
 collections:
   - name: community.general
   - name: kubernetes.core
-    version: 2.3.2
   - name: operator_sdk.util
   - name: community.docker
-    version: 3.4.5
   - name: awx.awx

roles/backup/tasks/awx-cro.yml

@@ -31,5 +31,6 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir  }}/awx_object"
     content: "{{ awx_spec | to_yaml }}"

roles/backup/tasks/delete_backup.yml

@@ -3,5 +3,6 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       bash -c 'rm -rf {{ backup_dir  }}'

roles/backup/tasks/init.yml

@@ -77,10 +77,10 @@
   set_fact:
     _postgres_image: "{{ _custom_postgres_image | default(lookup('env', 'RELATED_IMAGE_AWX_POSTGRES')) | default(_default_postgres_image, true) }}"
 
-- name: Create management pod from templated deployment config
+- name: Create management pod from the template
   k8s:
     name: "{{ ansible_operator_meta.name }}-db-management"
-    kind: Deployment
+    kind: Pod
     state: present
     definition: "{{ lookup('template', 'management-pod.yml.j2') }}"
     wait: true

roles/backup/tasks/postgres.yml

@@ -60,6 +60,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       mkdir -p {{ backup_dir }}
 
@@ -67,6 +68,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       touch {{ backup_dir }}/tower.db
 
@@ -126,6 +128,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: |
       bash -c "
       function end_keepalive {

roles/backup/tasks/secrets.yml

@@ -50,6 +50,7 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir }}/secrets.yml"
     content: "{{ secrets | to_yaml }}"
   no_log: "{{ no_log }}"

roles/installer/defaults/main.yml

@@ -13,6 +13,7 @@ ingress_class_name: ''
 ingress_path: '/'
 ingress_path_type: 'Prefix'
 ingress_api_version: 'networking.k8s.io/v1'
+api_urlpattern_prefix: ''
 # Add annotations to the service account. Specify as literal block. E.g.:
 # service_account_annotations: |
 #   eks.amazonaws.com/role-arn: arn:aws:iam::<ACCOUNT_ID>:role/<IAM_ROLE_NAME>
@@ -490,16 +491,3 @@ nginx_worker_processes: 1
 nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
 nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
-
-# metrics-utility (github.com/ansible/metrics-utility)
-_metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
-_metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
-_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
-_metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
-_metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
-_metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
-_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}"
-_metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
-_metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
-_metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"
-_metrics_utility_cronjob_report_schedule: "{{ metrics_utility_cronjob_report_schedule | default('@monthly') }}"

roles/installer/tasks/database_configuration.yml

@@ -100,6 +100,7 @@
     awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
     awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
     awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] |  default('prefer'|b64encode) | b64decode }}"
+    awx_postgres_target_session_attrs: "{{ pg_config['resources'][0]['data']['target_session_attrs'] | default('') | b64decode }}"
   no_log: "{{ no_log }}"
 
 - name: Set database as managed

roles/installer/tasks/resources_configuration.yml

@@ -267,8 +267,8 @@
           - status.phase=Running
       register: _new_pod
       until:
-        - "_new_pod['resources'] | length"
-        - "_new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | length"
+        - _new_pod['resources'] | length
+        - _new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | list | length
       retries: 60
       delay: 5
 

roles/installer/templates/configmaps/config.yaml.j2

@@ -54,6 +54,10 @@ data:
 
     INTERNAL_API_URL = 'http://127.0.0.1:8052'
 
+{% if api_urlpattern_prefix | length > 0 %}
+    OPTIONAL_API_URLPATTERN_PREFIX = '{{ api_urlpattern_prefix }}'
+{% endif %}
+
     # Container environments don't like chroots
     AWX_PROOT_ENABLED = False
 
@@ -184,11 +188,19 @@ data:
                 alias /var/lib/awx/public/static/;
             }
 
+            location {{ (ingress_path + '/locales').replace('//', '/') }} {
+                alias /var/lib/awx/public/static/awx/locales;
+            }
+
             location {{ (ingress_path + '/favicon.ico').replace('//', '/') }} {
                 alias /var/lib/awx/public/static/media/favicon.ico;
             }
 
+{% if api_urlpattern_prefix | length > 0 %}
+            location ~ ^({{ (ingress_path + '/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/' + api_urlpattern_prefix + '/v2/websocket/').replace('//', '/') }}) {
+{% else %}
             location ~ ^({{ (ingress_path + '/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/websocket/').replace('//', '/') }}) {
+{% endif %}
                 # Pass request to the upstream alias
                 proxy_pass http://daphne;
                 # Require http version 1.1 to allow for upgrade requests

roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2

@@ -59,6 +59,10 @@ spec:
               mountPath: "/etc/tower/conf.d/credentials.py"
               subPath: credentials.py
               readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
             - name: {{ ansible_operator_meta.name }}-settings
               mountPath: /etc/tower/settings.py
               subPath: settings.py
@@ -74,6 +78,12 @@ spec:
               items:
                 - key: credentials.py
                   path: 'credentials.py'
+          - name: "{{ secret_key_secret_name }}"
+            secret:
+              secretName: '{{ secret_key_secret_name }}'
+              items:
+                - key: secret_key
+                  path: SECRET_KEY
           - name: {{ ansible_operator_meta.name }}-settings
             configMap:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'

roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2

@@ -56,6 +56,10 @@ spec:
               mountPath: "/etc/tower/conf.d/credentials.py"
               subPath: credentials.py
               readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
             - name: {{ ansible_operator_meta.name }}-settings
               mountPath: /etc/tower/settings.py
               subPath: settings.py
@@ -71,6 +75,12 @@ spec:
               items:
                 - key: credentials.py
                   path: 'credentials.py'
+          - name: "{{ secret_key_secret_name }}"
+            secret:
+              secretName: '{{ secret_key_secret_name }}'
+              items:
+                - key: secret_key
+                  path: SECRET_KEY
           - name: {{ ansible_operator_meta.name }}-settings
             configMap:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'

roles/installer/templates/jobs/migration.yaml.j2

@@ -42,6 +42,34 @@ spec:
 {% for secret in image_pull_secrets %}
         - name: {{ secret }}
 {% endfor %}
+{% endif %}
+{% if task_node_selector %}
+      nodeSelector:
+        {{ task_node_selector | indent(width=8) }}
+{% elif node_selector %}
+      nodeSelector:
+        {{ node_selector | indent(width=8) }}
+{% endif %}
+{% if task_topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ task_topology_spread_constraints | indent(width=8) }}
+{% elif topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ topology_spread_constraints | indent(width=8) }}
+{% endif %}
+{% if task_tolerations %}
+      tolerations:
+        {{ task_tolerations | indent(width=8) }}
+{% elif tolerations %}
+      tolerations:
+        {{ tolerations | indent(width=8) }}
+{% endif %}
+{% if task_affinity %}
+      affinity:
+        {{ task_affinity | to_nice_yaml | indent(width=8) }}
+{% elif affinity %}
+      affinity:
+        {{ affinity | to_nice_yaml | indent(width=8) }}
 {% endif %}
       volumes:
         - name: "{{ ansible_operator_meta.name }}-application-credentials"

roles/installer/templates/settings/credentials.py.j2

@@ -10,6 +10,9 @@ DATABASES = {
         'OPTIONS': { 'sslmode': '{{ awx_postgres_sslmode }}',
 {% if awx_postgres_sslmode in ['verify-ca', 'verify-full'] %}
                      'sslrootcert': '{{ ca_trust_bundle }}',
+{% endif %}
+{% if awx_postgres_target_session_attrs %}
+                     'target_session_attrs': '{{ awx_postgres_target_session_attrs }}',
 {% endif %}
         },
     }
@@ -25,6 +28,9 @@ LISTENER_DATABASES = {
             'keepalives_count': {{ postgres_keepalives_count }},
 {% else %}
             'keepalives': 0,
+{% endif %}
+{% if awx_postgres_target_session_attrs %}
+            'target_session_attrs': '{{ awx_postgres_target_session_attrs }}',
 {% endif %}
         },
     }

roles/installer/vars/main.yml

@@ -8,3 +8,16 @@ supported_pg_version: 15
 _previous_upgraded_pg_version: 0
 old_postgres_pod: []
 _postgres_data_path: '/var/lib/pgsql/data/userdata'
+# metrics-utility (github.com/ansible/metrics-utility)
+_metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
+_metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
+_metrics_utility_secret: "{{ metrics_utility_secret | default('') }}"
+_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
+_metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
+_metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
+_metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
+_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}"
+_metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
+_metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
+_metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"
+_metrics_utility_cronjob_report_schedule: "{{ metrics_utility_cronjob_report_schedule | default('@monthly') }}"

roles/restore/defaults/main.yml

@@ -39,4 +39,6 @@ additional_labels: []
 
 # Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
 set_self_labels: true
+
+spec_overrides: {}
 ...

roles/restore/tasks/deploy_awx.yml

@@ -1,5 +1,10 @@
 ---
 
+- name: Combine spec_overrides with spec
+  set_fact:
+    spec: "{{ spec | default({}) | combine(spec_overrides) }}"
+  no_log: "{{ no_log }}"
+
 - name: Deploy AWX
   k8s:
     state: "{{ state | default('present') }}"

roles/restore/tasks/import_vars.yml

@@ -12,6 +12,7 @@
     k8s_cp:
       namespace: "{{ backup_pvc_namespace }}"
       pod: "{{ ansible_operator_meta.name }}-db-management"
+      container: "{{ ansible_operator_meta.name }}-db-management"
       remote_path: "{{ backup_dir  }}/awx_object"
       local_path: "{{ tmp_spec.path }}"
       state: from_pod

roles/restore/tasks/init.yml

@@ -86,10 +86,10 @@
   set_fact:
     _postgres_image: "{{ _custom_postgres_image | default(lookup('env', 'RELATED_IMAGE_AWX_POSTGRES'))  | default(_default_postgres_image, true) }}"
 
-- name: Create management pod from templated deployment config
+- name: Create management pod from the template
   k8s:
     name: "{{ ansible_operator_meta.name }}-db-management"
-    kind: Deployment
+    kind: Pod
     state: present
     definition: "{{ lookup('template', 'management-pod.yml.j2') }}"
     wait: true
@@ -98,6 +98,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       bash -c "stat {{ backup_dir }}"
   register: stat_backup_dir

roles/restore/tasks/postgres.yml

@@ -115,6 +115,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: |
       bash -c "
       function end_keepalive {

roles/restore/tasks/secrets.yml

@@ -10,6 +10,7 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir  }}/secrets.yml"
     local_path: "{{ tmp_secrets.path }}"
     state: from_pod