backup: Remove default parameter from jinja map (#1839)

The default paramater from the jinja map filter has been added in the
2.11.0 release.
However, the downstream ansible operator is still using ansible 2.9
with jinja 2.10.x so using the default parameter leads to the
following error:

TASK [Dump ingress tls secret names from awx spec and data into file]
********************************
The error was: jinja2.exceptions.FilterArgumentError: Unexpected
keyword argument 'default'
fatal: [localhost]: FAILED! => {
  "msg": "Unexpected failure during module execution.",
  "stdout": ""
}

Rather than using the default parameter with the map filter then add the
selectattr filter to get only the items with tls_secret defined and then
get the tls_secret attribute with the map filter.

This also gets rid of the when statement since we always get an empty
list when no tls_secret are present in ingress_hosts so the loop statement
will be skipped on the empty list.

Finally this changes the default value from the ingress_hosts field because
it's a list rather than a string.

https://jinja.palletsprojects.com/en/latest/templates/#jinja-filters.map

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>

.github/workflows/ci.yaml

@@ -17,6 +17,7 @@ jobs:
           - -t replicas
     env:
       DOCKER_API_VERSION: "1.41"
+      DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
     steps:
       - uses: actions/checkout@v3
 
@@ -37,10 +38,18 @@ jobs:
           MOLECULE_VERBOSITY: 3
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          STORE_DEBUG_OUTPUT: true
         run: |
           sudo rm -f $(which kustomize)
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }}
+
+      - name: Upload artifacts for failed tests if Run Molecule fails
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: awx_operator_molecule_test
+          path: ${{ env.DEBUG_OUTPUT_DIR }}
   helm:
     runs-on: ubuntu-latest
     name: helm

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1940,26 +1940,36 @@ spec:
                 description: Metrics-Utility Image PullPolicy
                 type: string
               metrics_utility_configmap:
-                description: Metrics-Utlity ConfigMap
+                description: Metrics-Utility ConfigMap
+                type: string
+              metrics_utility_secret:
+                description: Metrics-Utility Secret
                 type: string
               metrics_utility_cronjob_gather_schedule:
-                description: Metrics-Utlity Gather Data CronJob Schedule
+                description: Metrics-Utility Gather Data CronJob Schedule
                 type: string
                 default: '@hourly'
               metrics_utility_cronjob_report_schedule:
-                description: Metrics-Utlity Report CronJob Schedule
+                description: Metrics-Utility Report CronJob Schedule
                 type: string
                 default: '@monthly'
+              metrics_utility_ship_target:
+                description: Metrics-Utility Ship Target
+                type: string
               metrics_utility_pvc_claim:
-                description: Metrics-Utlity PVC Claim
+                description: Metrics-Utility PVC Claim
                 type: string
               metrics_utility_pvc_claim_size:
-                description: Metrics-Utlity PVC Claim Size
+                description: Metrics-Utility PVC Claim Size
                 type: string
                 default: 5Gi
               metrics_utility_pvc_claim_storage_class:
-                description: Metrics-Utlity PVC Claim Storage Class
+                description: Metrics-Utility PVC Claim Storage Class
                 type: string
+              metrics_utility_console_enabled:
+                description: Enable metrics utility shipping to Red Hat Hybrid Cloud Console
+                type: boolean
+                default: false
             type: object
           status:
             properties:

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -1039,7 +1039,7 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Image Version
+      - displayName: Metrics-Utility Image Version
         path: metrics_utility_image_version
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
@@ -1051,42 +1051,60 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:imagePullPolicy
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity ConfigMap
+      - displayName: Metrics-Utility ConfigMap
         path: metrics_utility_configmap
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:ConfigMap
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Gather Data CronJob Schedule
+      - displayName: Metrics-Utility Secret
+        path: metrics_utility_secret
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:Secret
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Gather Data CronJob Schedule
         path: metrics_utility_cronjob_gather_schedule
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Report CronJob Schedule
+      - displayName: Metrics-Utility Report CronJob Schedule
         path: metrics_utility_cronjob_report_schedule
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim
+      - displayName: Metrics-Utility Ship Target
+        path: metrics_utility_ship_target
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility PVC Claim
         path: metrics_utility_pvc_claim
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim Size
+      - displayName: Metrics-Utility PVC Claim Size
         path: metrics_utility_pvc_claim_size
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim Storage Class
+      - displayName: Metrics-Utility PVC Claim Storage Class
         path: metrics_utility_pvc_claim_storage_class
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Enabled Shipping to Red Hat Hybrid Cloud Console
+        path: metrics_utility_console_enabled
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
       statusDescriptors:
       - description: Route to access the instance deployed
         displayName: URL

docs/README.md

@@ -2,9 +2,15 @@
 
 To build the AWX Operator docs locally:
 
-1. Clone the AWX operator repository. 
-2. From the root directory:
-    a.  pip install --user -r docs/requirements.txt
-    b.  mkdocs build
+1. Clone the AWX operator repository.
+1. Preferrably, create a virtual environment for installing the dependencies.  
+   a. `python3 -m venv venv`  
+   b. `source venv/bin/activate`
+1. From the root directory:  
+   a. `pip install -r docs/requirements.txt`  
+   b. `mkdocs build`
+1. View the docs in your browser:  
+   a. `mkdocs serve`  
+   b. Open your browser and navigate to `http://127.0.0.1:8000/`
 
-This will create a new directory called `site/` in the root of your clone containing the index.html and static files. To view the docs in your browser, navigate there in your file explorer and double-click on the `index.html` file. This should open the docs site in your browser.
+This will create a new directory called `site/` in the root of your clone containing the index.html and static files.

docs/requirements.txt

@@ -50,6 +50,7 @@ idna==3.6
 jinja2==3.1.3
     # via
     #   mkdocs
+    #   mkdocs-macros-plugin
     #   mkdocs-material
     #   mkdocstrings
 jsmin==3.0.1
@@ -88,18 +89,21 @@ mkdocs==1.5.3
     #   mkdocs-autorefs
     #   mkdocs-gen-files
     #   mkdocs-htmlproofer-plugin
+    #   mkdocs-macros-plugin
     #   mkdocs-material
     #   mkdocs-minify-plugin
     #   mkdocs-monorepo-plugin
     #   mkdocstrings
-mkdocs-ansible==24.2.1
-    # via -r docs/requirements.in
+mkdocs-ansible==24.3.0
+    # via -r requirements.in
 mkdocs-autorefs==0.5.0
     # via mkdocstrings
 mkdocs-gen-files==0.5.0
     # via mkdocs-ansible
 mkdocs-htmlproofer-plugin==1.0.0
     # via mkdocs-ansible
+mkdocs-macros-plugin==1.0.5
+    # via mkdocs-ansible
 mkdocs-material==9.2.6
     # via mkdocs-ansible
 mkdocs-material-extensions==1.3.1
@@ -145,12 +149,15 @@ pymdown-extensions==10.0.1
 pyquery==2.0.0
     # via readtime
 python-dateutil==2.8.2
-    # via ghp-import
+    # via
+    #   ghp-import
+    #   mkdocs-macros-plugin
 python-slugify==8.0.4
     # via mkdocs-monorepo-plugin
 pyyaml==6.0.1
     # via
     #   mkdocs
+    #   mkdocs-macros-plugin
     #   pymdown-extensions
     #   pyyaml-env-tag
 pyyaml-env-tag==0.1
@@ -168,6 +175,8 @@ six==1.16.0
     # via python-dateutil
 soupsieve==2.5
     # via beautifulsoup4
+termcolor==2.4.0
+    # via mkdocs-macros-plugin
 text-unidecode==1.3
     # via python-slugify
 tinycss2==1.2.1

docs/upgrade/upgrading.md

@@ -1,6 +1,7 @@
 ### Upgrading
 
-To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX.  To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command
+To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX. To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command
+
 ```shell
 AWX_OPERATOR_VERSION=2.8.0
 docker run --entrypoint="" quay.io/ansible/awx-operator:$AWX_OPERATOR_VERSION bash -c "env | grep DEFAULT_AWX_VERSION"
@@ -12,26 +13,23 @@ Apply the awx-operator.yml for that release to upgrade the operator, and in turn
 
 The first part of any upgrade should be a backup. Note, there are secrets in the pod which work in conjunction with the database. Having just a database backup without the required secrets will not be sufficient for recovering from an issue when upgrading to a new version. See the [backup role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/backup) for information on how to backup your database and secrets.
 
-In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). *Before Restoring from a backup*, be sure to:
-* delete the old existing AWX CR
-* delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
+In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). _Before Restoring from a backup_, be sure to:
+
+- delete the old existing AWX CR
+- delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
 
 **Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well.
 
-
 #### PostgreSQL Upgrade Considerations
 
 If there is a PostgreSQL major version upgrade, after the data directory on the PVC is migrated to the new version, the old PVC is kept by default.
-This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically
-after a successful upgrade by setting the following variable on the AWX spec. 
-
+This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically after a successful upgrade by setting the following variable on the AWX spec.
 
 ```yaml
-  spec:
-    postgres_keep_pvc_after_upgrade: False
+spec:
+  postgres_keep_pvc_after_upgrade: False
 ```
 
-
 #### v0.14.0
 
 ##### Cluster-scope to Namespace-scope considerations

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -13,7 +13,8 @@ In a scenario where custom volumes and volume mounts are required to either over
 | init_container_extra_commands      | Specify additional commands for Init container           | ''      |
 
 
-> :warning: The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
+!!! warning
+    The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
 
 Example configuration for ConfigMap
 
@@ -26,62 +27,68 @@ metadata:
   namespace: <target namespace>
 data:
   ansible.cfg: |
-     [defaults]
-     remote_tmp = /tmp
-     [ssh_connection]
-     ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
+    [defaults]
+    remote_tmp = /tmp
+    [ssh_connection]
+    ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
   custom.py:  |
-      INSIGHTS_URL_BASE = "example.org"
-      AWX_CLEANUP_PATHS = True
+    INSIGHTS_URL_BASE = "example.org"
+    AWX_CLEANUP_PATHS = True
 ```
 Example spec file for volumes and volume mounts
 
 ```yaml
 ---
-    spec:
-    ...
-      extra_volumes: |
-        - name: ansible-cfg
-          configMap:
-            defaultMode: 420
-            items:
-              - key: ansible.cfg
-                path: ansible.cfg
-            name: <resourcename>-extra-config
-        - name: custom-py
-          configMap:
-            defaultMode: 420
-            items:
-              - key: custom.py
-                path: custom.py
-            name: <resourcename>-extra-config
-        - name: shared-volume
-          persistentVolumeClaim:
-            claimName: my-external-volume-claim
+spec:
+  ...
+  extra_volumes: |
+    - name: ansible-cfg
+      configMap:
+        defaultMode: 420
+        items:
+          - key: ansible.cfg
+            path: ansible.cfg
+        name: <resourcename>-extra-config
+    - name: custom-py
+      configMap:
+        defaultMode: 420
+        items:
+          - key: custom.py
+            path: custom.py
+        name: <resourcename>-extra-config
+    - name: shared-volume
+      persistentVolumeClaim:
+        claimName: my-external-volume-claim
 
-      init_container_extra_volume_mounts: |
-        - name: shared-volume
-          mountPath: /shared
+  init_container_extra_volume_mounts: |
+    - name: shared-volume
+      mountPath: /shared
 
-      init_container_extra_commands: |
-        # set proper permissions (rwx) for the awx user
-        chmod 775 /shared
-        chgrp 1000 /shared
+  init_container_extra_commands: |
+    # set proper permissions (rwx) for the awx user
+    chmod 775 /shared
+    chgrp 1000 /shared
 
-      ee_extra_volume_mounts: |
-        - name: ansible-cfg
-          mountPath: /etc/ansible/ansible.cfg
-          subPath: ansible.cfg
+  ee_extra_volume_mounts: |
+    - name: ansible-cfg
+      mountPath: /etc/ansible/ansible.cfg
+      subPath: ansible.cfg
 
-      task_extra_volume_mounts: |
-        - name: custom-py
-          mountPath: /etc/tower/conf.d/custom.py
-          subPath: custom.py
-        - name: shared-volume
-          mountPath: /shared
+  web_extra_volume_mounts: |
+    - name: custom-py
+      mountPath: /etc/tower/conf.d/custom.py
+      subPath: custom.py
+
+  task_extra_volume_mounts: |
+    - name: custom-py
+      mountPath: /etc/tower/conf.d/custom.py
+      subPath: custom.py
+    - name: shared-volume
+      mountPath: /shared
 ```
 
-> :warning: **Volume and VolumeMount names cannot contain underscores(_)**
+!!! warning
+    **Volume and VolumeMount names cannot contain underscores(_)**
 
 ##### Custom UWSGI Configuration
 We allow the customization of two UWSGI parameters:
@@ -143,7 +150,9 @@ $ oc create configmap favicon-configmap --from-file favicon.ico
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
 
 ```yaml
+---
 spec:
+  ...
   extra_volumes: |
     - name: favicon
       configMap:

docs/user-guide/advanced-configuration/extra-settings.md

@@ -24,3 +24,7 @@ Example configuration of `extra_settings` parameter
 ```
 
 Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes.
+
+!!! tip
+    Alternatively, you can pass any additional settings by mounting ConfigMaps or Secrets of the python files (`*.py`) that contain custom settings to under `/etc/tower/conf.d/` in the web and task pods.
+    See the example of `custom.py` in the [Custom Volume and Volume Mount Options](custom-volume-and-volume-mount-options.md) section.

molecule/default/molecule.yml

@@ -21,6 +21,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         default_awx_version: "{{ lookup('url', 'https://api.github.com/repos/ansible/awx/releases/latest') | from_json | json_query('tag_name') }}"
@@ -30,6 +31,8 @@ provisioner:
         operator_image: ${OPERATOR_IMAGE:-""}
         operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"}
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"}
 verifier:

molecule/default/tasks/apply_awx_spec.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create or update the awx.ansible.com/v1alpha1.AWX
+- name: Create or update the awx.ansible.com/v1beta1.AWX
   k8s:
     state: present
     namespace: '{{ namespace }}'

molecule/default/tasks/awx_test.yml

@@ -1,98 +1,8 @@
 ---
 - include_tasks: apply_awx_spec.yml
 
-- name: Obtain generated admin password
-  k8s_info:
-    namespace: '{{ namespace }}'
-    kind: Secret
-    name: example-awx-admin-password
-  register: admin_pw_secret
-
-- block:
-    - name: Get web pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-web
-      register: awx_web_pod
-      when: not awx_version
-
-    - name: Get task pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-task
-      register: awx_task_pod
-      when: not awx_version
-
-    - name: Extract tags from images from web pod
-      set_fact:
-        web_image_tags: |
-          {{ awx_web_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - name: Extract tags from images from task pod
-      set_fact:
-        task_image_tags: |
-          {{ awx_task_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - fail:
-        msg: |
-          It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
-          This is an environment variable that is set via build arg when releasing awx-operator.
-      when:
-        - not awx_version
-        - default_awx_version not in web_image_tags
-        - default_awx_version not in task_image_tags
-
-    - name: Launch Demo Job Template
-      awx.awx.job_launch:
-        name: Demo Job Template
-        wait: yes
-        validate_certs: no
-        controller_host: localhost/awx/
-        controller_username: admin
-        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-  rescue:
-    - name: Get list of project updates and jobs
-      uri:
-        url: "http://localhost/awx/api/v2/{{ resource }}/"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      register: job_lists
-      loop:
-        - project_updates
-        - jobs
-      loop_control:
-        loop_var: resource
-
-    - name: Get all job and project details
-      uri:
-        url: "http://localhost{{ endpoint }}"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      loop: |
-        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
-      loop_control:
-        loop_var: endpoint
-
-    - name: Re-emit failure
-      vars:
-        failed_task:
-          result: '{{ ansible_failed_result }}'
-      fail:
-        msg: '{{ failed_task }}'
-
-- block:
+- name: Validate AWX deployment
+  block:
     - name: Look up details for this AWX instance
       k8s_info:
         namespace: "{{ namespace }}"
@@ -117,65 +27,90 @@
           - app.kubernetes.io/name = example-awx-task
       register: awx_task_pod
 
-    - name: Extract additional_labels from AWX spec
-      set_fact:
-        awx_additional_labels: >-
-          {{ this_awx.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
+    - name: Validate DEFAULT_AWX_VERSION
+      block:
+      - name: Extract tags from images from web pod
+        set_fact:
+          web_image_tags: |
+            {{ awx_web_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - name: Extract tags from images from task pod
+        set_fact:
+          task_image_tags: |
+            {{ awx_task_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - fail:
+          msg: |
+            It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
+            This is an environment variable that is set via build arg when releasing awx-operator.
+        when:
+          - default_awx_version not in web_image_tags
+          - default_awx_version not in task_image_tags
+      when: not awx_version
 
-    - name: Extract additional_labels from AWX web Pod
-      set_fact:
-        awx_web_pod_additional_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
-
-    - name: Extract additional_labels from AWX task Pod
-      set_fact:
-        awx_task_pod_additional_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
+    - name: Validate additional_labels
+      block:
+      - name: Extract additional_labels from AWX spec
+        set_fact:
+          awx_additional_labels: >-
+            {{ this_awx.resources[0].metadata.labels
               | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
               | list
-          }}
+            }}
 
-    - name: Assert AWX web Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX web Pod
+        set_fact:
+          awx_web_pod_additional_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+              | list
+            }}
 
-    - name: Assert AWX task Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX task Pod
+        set_fact:
+          awx_task_pod_additional_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+                | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+                | list
+            }}
 
-    - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_web_pod_extra_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Assert AWX web Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_additional_labels == awx_additional_labels
 
-    - name: AWX web Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_extra_labels == []
+      - name: Assert AWX task Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_additional_labels == awx_additional_labels
 
-    - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_task_pod_extra_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_web_pod_extra_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
 
-    - name: AWX task Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_extra_labels == []
+      - name: AWX web Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_extra_labels == []
+
+      - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_task_pod_extra_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
+
+      - name: AWX task Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_extra_labels == []
 
   rescue:
     - name: Re-emit failure
@@ -184,3 +119,76 @@
           result: '{{ ansible_failed_result }}'
       fail:
         msg: '{{ failed_task }}'
+
+- name: Obtain generated admin password
+  k8s_info:
+    namespace: '{{ namespace }}'
+    kind: Secret
+    name: example-awx-admin-password
+  register: admin_pw_secret
+
+- name: Validate demo job launch
+  block:
+    - name: Launch Demo Job Template
+      awx.awx.job_launch:
+        name: Demo Job Template
+        wait: yes
+        validate_certs: no
+        controller_host: localhost/awx/
+        controller_username: admin
+        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+
+  rescue:
+    - name: Get list of project updates and jobs
+      uri:
+        url: "http://localhost/awx/api/v2/{{ resource }}/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: job_lists
+      loop:
+        - project_updates
+        - jobs
+      loop_control:
+        loop_var: resource
+
+    - name: Store job_lists debug output
+      copy:
+        content: "{{ job_lists | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_lists.json"
+      when: store_debug_output | default(false)
+
+    - name: Get all job and project details
+      uri:
+        url: "http://localhost{{ endpoint }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
+      loop_control:
+        loop_var: endpoint
+      register: job_details
+
+    - name: Store job_details debug output
+      copy:
+        content: "{{ job_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_details.json"
+      when: store_debug_output | default(false)
+
+    ## TODO: figure out why this doesn't work
+    # - name: Store debug outputs
+    #   copy:
+    #     content: '{{ item }}'
+    #     dest: "{{ debug_output_dir }}/{{ item }}.json"
+    #   loop:
+    #     - job_lists
+    #     - job_details
+    #   when: store_debug_output | default(false)
+
+    - name: Re-emit failure
+      vars:
+        failed_task:
+          result: '{{ ansible_failed_result }}'
+      fail:
+        msg: '{{ failed_task }}'

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -13,6 +13,12 @@ spec:
 {% endif %}
 {% if awx_version %}
   image_version: {{ awx_version }}
+{% endif %}
+{% if awx_ee_image %}
+  control_plane_ee_image: {{ awx_ee_image }}
+  ee_images:
+  - image: {{ awx_ee_image }}
+    name: AWX EE
 {% endif %}
   ingress_type: ingress
   ingress_path: /awx

molecule/default/utils/output_all_container_logs_for_pod.yml

@@ -0,0 +1,15 @@
+---
+- name: Get all container log in pod
+  kubernetes.core.k8s_log:
+    namespace: '{{ namespace }}'
+    name: '{{ item.metadata.name }}'
+    all_containers: true
+  register: all_container_logs
+
+- name: Store logs in file
+  ansible.builtin.copy:
+    content: "{{ all_container_logs.log_lines | join('\n') }}"
+    dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log'
+
+# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs
+# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported

molecule/default/utils/output_k8s_resources.yml

@@ -0,0 +1,29 @@
+---
+- name: Retrieve relevant k8s resources
+  kubernetes.core.k8s_info:
+    api_version: '{{ item.api_version }}'
+    kind: '{{ item.kind }}'
+    namespace: '{{ namespace }}'
+  loop:
+    - api_version: v1
+      kind: Pod
+    - api_version: apps/v1
+      kind: Deployment
+    - api_version: v1
+      kind: Secret
+    - api_version: v1
+      kind: ConfigMap
+    - api_version: "awx.ansible.com/v1beta1"
+      kind: AWX
+  register: debug_resources
+
+- name: debug print item.kind and item.metadata.name
+  debug:
+    msg: '{{ item.kind }}-{{ item.metadata.name }}'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"
+
+- name: Output gathered resource to files
+  ansible.builtin.copy:
+    content: '{{ item | to_nice_json }}'
+    dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"

molecule/default/verify.yml

@@ -10,53 +10,41 @@
     ctrl_label: control-plane=controller-manager
 
   tasks:
-    - block:
+    - name: Perform awx tests
+      block:
         - name: Import all test files from tasks/
-          include_tasks: '{{ item }}'
+          ansible.builtin.include_tasks: '{{ item }}'
           with_fileglob:
             - tasks/awx_test.yml
             - tasks/awx_replicas_test.yml
           tags:
             - always
       rescue:
-        - name: Retrieve relevant resources
-          k8s_info:
-            api_version: '{{ item.api_version }}'
-            kind: '{{ item.kind }}'
+        - name: Create debug output directory
+          ansible.builtin.file:
+            path: '{{ debug_output_dir }}'
+            state: directory
+          tags:
+            - always
+
+        - name: Gather and output K8s resources
+          ansible.builtin.include_tasks: utils/output_k8s_resources.yml
+          tags:
+            - always
+
+        - name: Get all pods
+          kubernetes.core.k8s_info:
+            api_version: v1
+            kind: Pod
             namespace: '{{ namespace }}'
-          loop:
-            - api_version: v1
-              kind: Pod
-            - api_version: apps/v1
-              kind: Deployment
-            - api_version: v1
-              kind: Secret
-            - api_version: v1
-              kind: ConfigMap
-          register: debug_resources
+          register: all_pods
           tags:
             - always
 
-        - name: Retrieve Pod logs
-          k8s_log:
-            name: '{{ item.metadata.name }}'
-            namespace: '{{ namespace }}'
-            container: awx-manager
-          loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}"
-          register: debug_logs
-          tags:
-            - always
-
-        - name: Output gathered resources
-          debug:
-            var: debug_resources
-          tags:
-            - always
-
-        - name: Output gathered logs
-          debug:
-            var: item.log_lines
-          loop: '{{ debug_logs.results }}'
+        - name: Get all container logs for all pods
+          ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml
+          loop: '{{ all_pods.resources }}'
+          ignore_errors: yes
           tags:
             - always
 
@@ -64,7 +52,7 @@
           vars:
             failed_task:
               result: '{{ ansible_failed_result }}'
-          fail:
+          ansible.builtin.fail:
             msg: '{{ failed_task }}'
           tags:
             - always

molecule/kind/molecule.yml

@@ -23,6 +23,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         ansible_python_interpreter: '{{ ansible_playbook_python }}'
@@ -34,6 +35,8 @@ provisioner:
         operator_pull_policy: "Never"
         kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig
     KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig

roles/backup/tasks/secrets.yml

@@ -20,9 +20,7 @@
 
 - name: Dump ingress tls secret names from awx spec and data into file
   include_tasks: dump_ingress_tls_secrets.yml
-  with_items:
-    - "{{ awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list }}"
-  when: awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list | length
+  with_items: "{{ awx_spec.spec['ingress_hosts'] | default([]) | selectattr('tls_secret', 'defined') | map(attribute='tls_secret') | list }}"
 
 - name: Dump receptor secret names and data into file
   include_tasks: dump_receptor_secrets.yml

roles/installer/defaults/main.yml

@@ -494,9 +494,11 @@ nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
 # metrics-utility (github.com/ansible/metrics-utility)
 _metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
 _metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
+_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
 _metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
 _metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
 _metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
+_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}"
 _metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
 _metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
 _metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"

roles/installer/tasks/enable_metrics_utility.yml

@@ -1,23 +1,42 @@
 ---
-# Check to make sure provided pvc exists, error loudly if not.  Otherwise, the management pod will just stay in pending state forever.
-- name: Check provided PVC claim exists
-  kubernetes.core.k8s_info:
-    name: "{{ _metrics_utility_pvc_claim }}"
-    kind: PersistentVolumeClaim
-    namespace: "{{ ansible_operator_meta.namespace }}"
-  when:
-    - _metrics_utility_pvc_claim | length
+- name: Setup PVC if using directory ship target
+  block:
 
-- name: Create PVC for metrics-utility
-  kubernetes.core.k8s:
-    kind: PersistentVolumeClaim
-    definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+    # Check to make sure provided pvc exists
+    - name: Check provided PVC claim exists
+      kubernetes.core.k8s_info:
+        name: "{{ _metrics_utility_pvc_claim }}"
+        kind: PersistentVolumeClaim
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      when:
+        - _metrics_utility_pvc_claim | length
 
-- name: Create Kubernetes CronJobs for metrics-utility
+    - name: Create PVC for metrics-utility
+      kubernetes.core.k8s:
+        kind: PersistentVolumeClaim
+        definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+
+  when: _metrics_utility_ship_target == "directory"
+
+- name: Create default metrics-utility Kubernetes CronJobs
   kubernetes.core.k8s:
-    definition: "{{ lookup('template', item) }}"
+    definition: "{{ lookup('template', item.template) }}"
     apply: true
     wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
   loop:
-    - cronjobs/metrics-utility-gather.yaml.j2
-    - cronjobs/metrics-utility-report.yaml.j2
+    - {name: 'metrics-utility-gather', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+    - {name: 'metrics-utility-report', template: 'cronjobs/metrics-utility-report.yaml.j2'}
+
+- name: Create metrics-utility Kubernetes CronJob for Red Hat Hybrid Cloud Console
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', item.template) }}"
+    apply: true
+    wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
+    metrics_utility_ship_target: crc  # TODO - Update to console when changed
+  loop:
+    - {name: 'metrics-utility-gather-console', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+  when: _metrics_utility_console_enabled

roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2

@@ -2,10 +2,10 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+  name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
     {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
@@ -16,7 +16,7 @@ spec:
       template:
         metadata:
           labels:
-            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
             {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
             {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
         spec:
@@ -30,7 +30,7 @@ spec:
 {% endfor %}
 {% endif %}
           containers:
-          - name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+          - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
             image: "{{ _metrics_utility_image }}"
             imagePullPolicy: "{{ image_pull_policy }}"
             resources:
@@ -41,9 +41,16 @@ spec:
               - /bin/sh
               - -c
               - metrics-utility gather_automation_controller_billing_data --ship --until=10m
+            env:
+            - name: METRICS_UTILITY_SHIP_TARGET
+              value: "{{ _metrics_utility_ship_target }}"
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
+{% if _metrics_utility_secret is defined %}
+            - secretRef:
+                name: {{ _metrics_utility_secret }}
+{% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
               mountPath: /metrics-utility

roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2

@@ -2,10 +2,10 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: {{ ansible_operator_meta.name }}-metrics-utility-report
+  name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
     {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
@@ -16,7 +16,7 @@ spec:
       template:
         metadata:
           labels:
-            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
             {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
             {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
         spec:
@@ -30,7 +30,7 @@ spec:
 {% endfor %}
 {% endif %}
           containers:
-          - name: {{ ansible_operator_meta.name }}-metrics-utility-report
+          - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
             image: "{{ _metrics_utility_image }}"
             imagePullPolicy: "{{ image_pull_policy }}"
             resources:
@@ -44,6 +44,10 @@ spec:
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
+{% if _metrics_utility_secret is defined %}
+            - secretRef:
+                name: {{ _metrics_utility_secret }}
+{% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
               mountPath: /metrics-utility