backup: Remove default parameter from jinja map (#1839)

The default paramater from the jinja map filter has been added in the
2.11.0 release.
However, the downstream ansible operator is still using ansible 2.9
with jinja 2.10.x so using the default parameter leads to the
following error:

TASK [Dump ingress tls secret names from awx spec and data into file]
********************************
The error was: jinja2.exceptions.FilterArgumentError: Unexpected
keyword argument 'default'
fatal: [localhost]: FAILED! => {
  "msg": "Unexpected failure during module execution.",
  "stdout": ""
}

Rather than using the default parameter with the map filter then add the
selectattr filter to get only the items with tls_secret defined and then
get the tls_secret attribute with the map filter.

This also gets rid of the when statement since we always get an empty
list when no tls_secret are present in ingress_hosts so the loop statement
will be skipped on the empty list.

Finally this changes the default value from the ingress_hosts field because
it's a list rather than a string.

https://jinja.palletsprojects.com/en/latest/templates/#jinja-filters.map

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>

.github/workflows/ci.yaml

@@ -17,6 +17,7 @@ jobs:
           - -t replicas
     env:
       DOCKER_API_VERSION: "1.41"
+      DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
     steps:
       - uses: actions/checkout@v3
 
@@ -37,10 +38,18 @@ jobs:
           MOLECULE_VERBOSITY: 3
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          STORE_DEBUG_OUTPUT: true
         run: |
           sudo rm -f $(which kustomize)
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }}
+
+      - name: Upload artifacts for failed tests if Run Molecule fails
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: awx_operator_molecule_test
+          path: ${{ env.DEBUG_OUTPUT_DIR }}
   helm:
     runs-on: ubuntu-latest
     name: helm
@@ -99,3 +108,5 @@ jobs:
             echo "${no_log}"
             exit 1
           fi
+  nox-sessions:
+    uses: ./.github/workflows/reusable-nox.yml

.github/workflows/reusable-nox.yml

@@ -0,0 +1,26 @@
+---
+name: nox
+
+"on":
+    workflow_call:
+
+jobs:
+    nox:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                - session: build
+                  python-versions: "3.11"
+        name: "Run nox ${{ matrix.session }} session"
+        steps:
+        - name: Check out repo
+          uses: actions/checkout@v4
+        - name: Setup nox
+          uses: wntrblm/nox@2024.03.02
+          with:
+              python-versions: "${{ matrix.python-versions }}"
+        - name: "Run nox -s ${{ matrix.session }}"
+          run: |
+            nox -s "${{ matrix.session }}"

.helm/starter/templates/storage/postgres-pv.yaml

@@ -13,7 +13,7 @@ spec:
     storage: {{ default "8Gi" .size | quote }}
   storageClassName: {{ include "postgres.storageClassName" $ }}
   hostPath:
-    path: {{ required "customVolumes.postgres.hostPath or spec.postgres_data_path are required!" (default ($.Values.AWX.spec).postgres_data_path .hostPath) | quote }}
+    path: /var/lib/pgsql/data/userdata
+{{- end }}
 {{- end }}
 {{- end }}
-{{- end }}

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1460,7 +1460,7 @@ spec:
                     type: object
                 type: object
               postgres_init_container_resource_requirements:
-                description: Resource requirements for the postgres init container
+                description: (Deprecated, use postgres_resource_requirements parameter) Resource requirements for the postgres init container
                 properties:
                   requests:
                     properties:
@@ -1807,13 +1807,16 @@ spec:
               postgres_priority_class:
                 description: Assign a preexisting priority class to the postgres pod
                 type: string
-              postgres_data_path:
-                description: Path where the PostgreSQL data are located
-                type: string
               postgres_extra_args:
                 type: array
                 items:
                   type: string
+              postgres_data_volume_init:
+                description: Sets permissions on the /var/lib/pgdata/data for postgres container using an init container (not Openshift)
+                type: boolean
+              postgres_init_container_commands:
+                description: Customize the postgres init container commands (Non Openshift)
+                type: string
               postgres_extra_volumes:
                 description: Specify extra volumes to add to the application pod
                 type: string
@@ -1937,26 +1940,36 @@ spec:
                 description: Metrics-Utility Image PullPolicy
                 type: string
               metrics_utility_configmap:
-                description: Metrics-Utlity ConfigMap
+                description: Metrics-Utility ConfigMap
+                type: string
+              metrics_utility_secret:
+                description: Metrics-Utility Secret
                 type: string
               metrics_utility_cronjob_gather_schedule:
-                description: Metrics-Utlity Gather Data CronJob Schedule
+                description: Metrics-Utility Gather Data CronJob Schedule
                 type: string
                 default: '@hourly'
               metrics_utility_cronjob_report_schedule:
-                description: Metrics-Utlity Report CronJob Schedule
+                description: Metrics-Utility Report CronJob Schedule
                 type: string
                 default: '@monthly'
+              metrics_utility_ship_target:
+                description: Metrics-Utility Ship Target
+                type: string
               metrics_utility_pvc_claim:
-                description: Metrics-Utlity PVC Claim
+                description: Metrics-Utility PVC Claim
                 type: string
               metrics_utility_pvc_claim_size:
-                description: Metrics-Utlity PVC Claim Size
+                description: Metrics-Utility PVC Claim Size
                 type: string
                 default: 5Gi
               metrics_utility_pvc_claim_storage_class:
-                description: Metrics-Utlity PVC Claim Storage Class
+                description: Metrics-Utility PVC Claim Storage Class
                 type: string
+              metrics_utility_console_enabled:
+                description: Enable metrics utility shipping to Red Hat Hybrid Cloud Console
+                type: boolean
+                default: false
             type: object
           status:
             properties:

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -456,12 +456,21 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
-      - description: The PostgreSQL init container is not used when an external DB
-          is configured
+      - description: Sets permissions on the /var/lib/pgsql/data for postgres container using an init container (not Openshift)
+        displayName: PostgreSQL initialize data volume
+        path: postgres_data_volume_init
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: Customize the postgres init container commands (Non Openshift)
+        displayName: PostgreSQL Init Container Commands
+        path: postgres_init_container_commands
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: (Deprecated, use postgres_resource_requirements parameter instead)
         displayName: PostgreSQL Init Container Resource Requirements
         path: postgres_init_container_resource_requirements
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
       - displayName: Redis Container Resource Requirements
         path: redis_resource_requirements
@@ -637,11 +646,6 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:advanced
-      - displayName: Postgres Datapath
-        path: postgres_data_path
-        x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
-        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Postgres Extra Arguments
         path: postgres_extra_args
         x-descriptors:
@@ -1034,7 +1038,8 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
-      - displayName: Metrics-Utlity Image Version
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Image Version
         path: metrics_utility_image_version
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
@@ -1046,42 +1051,60 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:imagePullPolicy
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity ConfigMap
+      - displayName: Metrics-Utility ConfigMap
         path: metrics_utility_configmap
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:ConfigMap
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Gather Data CronJob Schedule
+      - displayName: Metrics-Utility Secret
+        path: metrics_utility_secret
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:Secret
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Gather Data CronJob Schedule
         path: metrics_utility_cronjob_gather_schedule
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Report CronJob Schedule
+      - displayName: Metrics-Utility Report CronJob Schedule
         path: metrics_utility_cronjob_report_schedule
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim
+      - displayName: Metrics-Utility Ship Target
+        path: metrics_utility_ship_target
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility PVC Claim
         path: metrics_utility_pvc_claim
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim Size
+      - displayName: Metrics-Utility PVC Claim Size
         path: metrics_utility_pvc_claim_size
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim Storage Class
+      - displayName: Metrics-Utility PVC Claim Storage Class
         path: metrics_utility_pvc_claim_storage_class
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Enabled Shipping to Red Hat Hybrid Cloud Console
+        path: metrics_utility_console_enabled
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
       statusDescriptors:
       - description: Route to access the instance deployed
         displayName: URL

config/samples/awx_v1beta1_awx_resource_limits.yaml

@@ -46,10 +46,3 @@ spec:
     limits:
       cpu: 1000m
       memory: 2Gi
-  postgres_init_container_resource_requirements:
-    requests:
-      cpu: 10m
-      memory: 64Mi
-    limits:
-      cpu: 1000m
-      memory: 2Gi

docs/README.md

@@ -2,9 +2,15 @@
 
 To build the AWX Operator docs locally:
 
-1. Clone the AWX operator repository. 
-2. From the root directory:
-    a.  pip install --user -r docs/requirements.txt
-    b.  mkdocs build
+1. Clone the AWX operator repository.
+1. Preferrably, create a virtual environment for installing the dependencies.  
+   a. `python3 -m venv venv`  
+   b. `source venv/bin/activate`
+1. From the root directory:  
+   a. `pip install -r docs/requirements.txt`  
+   b. `mkdocs build`
+1. View the docs in your browser:  
+   a. `mkdocs serve`  
+   b. Open your browser and navigate to `http://127.0.0.1:8000/`
 
-This will create a new directory called `site/` in the root of your clone containing the index.html and static files. To view the docs in your browser, navigate there in your file explorer and double-click on the `index.html` file. This should open the docs site in your browser.
+This will create a new directory called `site/` in the root of your clone containing the index.html and static files.

docs/requirements.txt

@@ -50,6 +50,7 @@ idna==3.6
 jinja2==3.1.3
     # via
     #   mkdocs
+    #   mkdocs-macros-plugin
     #   mkdocs-material
     #   mkdocstrings
 jsmin==3.0.1
@@ -88,18 +89,21 @@ mkdocs==1.5.3
     #   mkdocs-autorefs
     #   mkdocs-gen-files
     #   mkdocs-htmlproofer-plugin
+    #   mkdocs-macros-plugin
     #   mkdocs-material
     #   mkdocs-minify-plugin
     #   mkdocs-monorepo-plugin
     #   mkdocstrings
-mkdocs-ansible==24.2.1
-    # via -r docs/requirements.in
+mkdocs-ansible==24.3.0
+    # via -r requirements.in
 mkdocs-autorefs==0.5.0
     # via mkdocstrings
 mkdocs-gen-files==0.5.0
     # via mkdocs-ansible
 mkdocs-htmlproofer-plugin==1.0.0
     # via mkdocs-ansible
+mkdocs-macros-plugin==1.0.5
+    # via mkdocs-ansible
 mkdocs-material==9.2.6
     # via mkdocs-ansible
 mkdocs-material-extensions==1.3.1
@@ -145,12 +149,15 @@ pymdown-extensions==10.0.1
 pyquery==2.0.0
     # via readtime
 python-dateutil==2.8.2
-    # via ghp-import
+    # via
+    #   ghp-import
+    #   mkdocs-macros-plugin
 python-slugify==8.0.4
     # via mkdocs-monorepo-plugin
 pyyaml==6.0.1
     # via
     #   mkdocs
+    #   mkdocs-macros-plugin
     #   pymdown-extensions
     #   pyyaml-env-tag
 pyyaml-env-tag==0.1
@@ -168,6 +175,8 @@ six==1.16.0
     # via python-dateutil
 soupsieve==2.5
     # via beautifulsoup4
+termcolor==2.4.0
+    # via mkdocs-macros-plugin
 text-unidecode==1.3
     # via python-slugify
 tinycss2==1.2.1

docs/upgrade/upgrading.md

@@ -1,6 +1,7 @@
 ### Upgrading
 
-To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX.  To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command
+To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX. To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command
+
 ```shell
 AWX_OPERATOR_VERSION=2.8.0
 docker run --entrypoint="" quay.io/ansible/awx-operator:$AWX_OPERATOR_VERSION bash -c "env | grep DEFAULT_AWX_VERSION"
@@ -12,26 +13,23 @@ Apply the awx-operator.yml for that release to upgrade the operator, and in turn
 
 The first part of any upgrade should be a backup. Note, there are secrets in the pod which work in conjunction with the database. Having just a database backup without the required secrets will not be sufficient for recovering from an issue when upgrading to a new version. See the [backup role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/backup) for information on how to backup your database and secrets.
 
-In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). *Before Restoring from a backup*, be sure to:
-* delete the old existing AWX CR
-* delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
+In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). _Before Restoring from a backup_, be sure to:
+
+- delete the old existing AWX CR
+- delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
 
 **Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well.
 
-
 #### PostgreSQL Upgrade Considerations
 
 If there is a PostgreSQL major version upgrade, after the data directory on the PVC is migrated to the new version, the old PVC is kept by default.
-This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically
-after a successful upgrade by setting the following variable on the AWX spec. 
-
+This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically after a successful upgrade by setting the following variable on the AWX spec.
 
 ```yaml
-  spec:
-    postgres_keep_pvc_after_upgrade: False
+spec:
+  postgres_keep_pvc_after_upgrade: False
 ```
 
-
 #### v0.14.0
 
 ##### Cluster-scope to Namespace-scope considerations

docs/user-guide/advanced-configuration/container-probes.md

@@ -2,6 +2,17 @@
 These parameters control the usage of liveness and readiness container probes for
 the web and task containers.
 
+> [!ALERT]
+> All of probes are disabled by default for now, to enable it, set the *_period parameters.  For example:
+
+```
+
+web_liveness_period: 15
+web_readiness_period: 15
+task_liveness_period: 15
+task_readiness_period: 15
+```
+
 #### Web / Task Container Liveness Check
 
 The liveness probe queries the status of the supervisor daemon of the container.  The probe will fail if it

docs/user-guide/advanced-configuration/containers-resource-requirements.md

@@ -22,16 +22,15 @@ spec:
 
 The resource requirements for both, the task and the web containers are configurable - both the lower end (requests) and the upper end (limits).
 
-| Name                       | Description                                      | Default                              |
-| -------------------------- | ------------------------------------------------ | ------------------------------------ |
-| web_resource_requirements  | Web container resource requirements              | requests: {cpu: 100m, memory: 128Mi} |
-| task_resource_requirements | Task container resource requirements             | requests: {cpu: 100m, memory: 128Mi} |
-| ee_resource_requirements   | EE control plane container resource requirements | requests: {cpu: 50m, memory: 64Mi} |
-| redis_resource_requirements   | Redis container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
-| postgres_resource_requirements   | Postgres container resource requirements | requests: {cpu: 10m, memory: 64Mi} |
-| rsyslog_resource_requirements   | Rsyslog container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
-| init_container_resource_requirements   | Init Container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
-| postgres_init_container_resource_requirements   | Postgres Init Container resource requirements | requests: {cpu: 10m, memory: 64Mi} |
+| Name                                 | Description                                                  | Default                              |
+| ------------------------------------ | ------------------------------------------------------------ | ------------------------------------ |
+| web_resource_requirements            | Web container resource requirements                          | requests: {cpu: 100m, memory: 128Mi} |
+| task_resource_requirements           | Task container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
+| ee_resource_requirements             | EE control plane container resource requirements             | requests: {cpu: 50m, memory: 64Mi} |
+| redis_resource_requirements          | Redis container resource requirements                        | requests: {cpu: 100m, memory: 128Mi} |
+| postgres_resource_requirements       | Postgres container (and initContainer) resource requirements | requests: {cpu: 10m, memory: 64Mi}   |
+| rsyslog_resource_requirements        | Rsyslog container resource requirements                      | requests: {cpu: 100m, memory: 128Mi} |
+| init_container_resource_requirements | Init Container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
 
 
 Example of customization could be:
@@ -85,13 +84,6 @@ spec:
     limits:
       cpu: 1000m
       memory: 2Gi
-  postgres_init_container_resource_requirements:
-    requests:
-      cpu: 10m
-      memory: 64Mi
-    limits:
-      cpu: 1000m
-      memory: 2Gi
 ```
 
 

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -13,7 +13,8 @@ In a scenario where custom volumes and volume mounts are required to either over
 | init_container_extra_commands      | Specify additional commands for Init container           | ''      |
 
 
-> :warning: The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
+!!! warning
+    The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
 
 Example configuration for ConfigMap
 
@@ -26,62 +27,68 @@ metadata:
   namespace: <target namespace>
 data:
   ansible.cfg: |
-     [defaults]
-     remote_tmp = /tmp
-     [ssh_connection]
-     ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
+    [defaults]
+    remote_tmp = /tmp
+    [ssh_connection]
+    ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
   custom.py:  |
-      INSIGHTS_URL_BASE = "example.org"
-      AWX_CLEANUP_PATHS = True
+    INSIGHTS_URL_BASE = "example.org"
+    AWX_CLEANUP_PATHS = True
 ```
 Example spec file for volumes and volume mounts
 
 ```yaml
 ---
-    spec:
-    ...
-      extra_volumes: |
-        - name: ansible-cfg
-          configMap:
-            defaultMode: 420
-            items:
-              - key: ansible.cfg
-                path: ansible.cfg
-            name: <resourcename>-extra-config
-        - name: custom-py
-          configMap:
-            defaultMode: 420
-            items:
-              - key: custom.py
-                path: custom.py
-            name: <resourcename>-extra-config
-        - name: shared-volume
-          persistentVolumeClaim:
-            claimName: my-external-volume-claim
+spec:
+  ...
+  extra_volumes: |
+    - name: ansible-cfg
+      configMap:
+        defaultMode: 420
+        items:
+          - key: ansible.cfg
+            path: ansible.cfg
+        name: <resourcename>-extra-config
+    - name: custom-py
+      configMap:
+        defaultMode: 420
+        items:
+          - key: custom.py
+            path: custom.py
+        name: <resourcename>-extra-config
+    - name: shared-volume
+      persistentVolumeClaim:
+        claimName: my-external-volume-claim
 
-      init_container_extra_volume_mounts: |
-        - name: shared-volume
-          mountPath: /shared
+  init_container_extra_volume_mounts: |
+    - name: shared-volume
+      mountPath: /shared
 
-      init_container_extra_commands: |
-        # set proper permissions (rwx) for the awx user
-        chmod 775 /shared
-        chgrp 1000 /shared
+  init_container_extra_commands: |
+    # set proper permissions (rwx) for the awx user
+    chmod 775 /shared
+    chgrp 1000 /shared
 
-      ee_extra_volume_mounts: |
-        - name: ansible-cfg
-          mountPath: /etc/ansible/ansible.cfg
-          subPath: ansible.cfg
+  ee_extra_volume_mounts: |
+    - name: ansible-cfg
+      mountPath: /etc/ansible/ansible.cfg
+      subPath: ansible.cfg
 
-      task_extra_volume_mounts: |
-        - name: custom-py
-          mountPath: /etc/tower/conf.d/custom.py
-          subPath: custom.py
-        - name: shared-volume
-          mountPath: /shared
+  web_extra_volume_mounts: |
+    - name: custom-py
+      mountPath: /etc/tower/conf.d/custom.py
+      subPath: custom.py
+
+  task_extra_volume_mounts: |
+    - name: custom-py
+      mountPath: /etc/tower/conf.d/custom.py
+      subPath: custom.py
+    - name: shared-volume
+      mountPath: /shared
 ```
 
-> :warning: **Volume and VolumeMount names cannot contain underscores(_)**
+!!! warning
+    **Volume and VolumeMount names cannot contain underscores(_)**
 
 ##### Custom UWSGI Configuration
 We allow the customization of two UWSGI parameters:
@@ -143,7 +150,9 @@ $ oc create configmap favicon-configmap --from-file favicon.ico
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
 
 ```yaml
+---
 spec:
+  ...
   extra_volumes: |
     - name: favicon
       configMap:

docs/user-guide/advanced-configuration/deploying-a-specific-version-of-awx.md

@@ -2,15 +2,19 @@
 
 There are a few variables that are customizable for awx the image management.
 
-| Name                | Description               | Default                                    |
-| ------------------- | ------------------------- | -----------------------------------------  |
-| image               | Path of the image to pull | quay.io/ansible/awx                        |
-| image_version       | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
-| image_pull_policy   | The pull policy to adopt  | IfNotPresent                               |
-| image_pull_secrets  | The pull secrets to use   | None                                       |
-| ee_images           | A list of EEs to register | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
-| redis_image         | Path of the image to pull | docker.io/redis                            |
-| redis_image_version | Image version to pull     | latest                                     |
+| Name                          | Description               | Default                                    |
+| ----------------------------- | ------------------------- | ------------------------------------------ |
+| image                         | Path of the image to pull | quay.io/ansible/awx                        |
+| image_version                 | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| image_pull_policy             | The pull policy to adopt  | IfNotPresent                               |
+| image_pull_secrets            | The pull secrets to use   | None                                       |
+| ee_images                     | A list of EEs to register | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| redis_image                   | Path of the image to pull | docker.io/redis                            |
+| redis_image_version           | Image version to pull     | latest                                     |
+| control_plane_ee_image        | Image version to pull     | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| init_container_image          | Path of the image to pull | quay.io/ansible/awx-ee                     |
+| init_container_image_version  | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| init_projects_container_image | Image version to pull     | quay.io/centos/centos:stream9              |
 
 Example of customization could be:
 
@@ -26,6 +30,10 @@ spec:
   ee_images:
     - name: my-custom-awx-ee
       image: myorg/my-custom-awx-ee
+  control_plane_ee_image: myorg/my-custom-awx-ee:latest
+  init_container_image: myorg/my-custom-awx-ee
+  init_container_image_version: latest
+  init_projects_container_image: myorg/my-mirrored-centos:stream9
 ```
 
 **Note**: The `image` and `image_version` are intended for local mirroring scenarios. Please note that using a version of AWX other than the one bundled with the `awx-operator` is **not** supported. For the default values, check the [main.yml](https://github.com/ansible/awx-operator/blob/devel/roles/installer/defaults/main.yml) file.

docs/user-guide/advanced-configuration/extra-settings.md

@@ -24,3 +24,7 @@ Example configuration of `extra_settings` parameter
 ```
 
 Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes.
+
+!!! tip
+    Alternatively, you can pass any additional settings by mounting ConfigMaps or Secrets of the python files (`*.py`) that contain custom settings to under `/etc/tower/conf.d/` in the web and task pods.
+    See the example of `custom.py` in the [Custom Volume and Volume Mount Options](custom-volume-and-volume-mount-options.md) section.

docs/user-guide/advanced-configuration/mesh-ingress.md

@@ -86,7 +86,7 @@ spec:
 !!! tip
     AWX Operator supports both API groups `traefik.io` and `traefik.containo.us` in `ingress_api_version` for Traefik, but it is recommended to use `traefik.io` since `traefik.containo.us` is deprecated in Traefik v2.10 and is removed in Traefik v3. Refer to [Traefik documentation](https://doc.traefik.io/traefik/migration/v2/#v210) for more information about deprecation.
 
-    If you can't see any IngressRouteTCP resources by `kubectl` command after deploying mesh ingress, you should fully qualify the resource name with API group, `kubectl get ingressroutetcp.traefik.io` or `kubectl get traefik.containo.us` for example.
+    If you can't see any IngressRouteTCP resources by `kubectl` command after deploying mesh ingress, you should fully qualify the resource name with API group, `kubectl get ingressroutetcp.traefik.io` or `kubectl get ingressroutetcp.traefik.containo.us` for example.
 
 ### On Kubernetes with User managed Ingress
 

docs/user-guide/database-configuration.md

@@ -56,15 +56,14 @@ If you don't have access to an external PostgreSQL service, the AWX operator can
 
 The following variables are customizable for the managed PostgreSQL service
 
-| Name                                          | Description                                   | Default                                 |
-| --------------------------------------------- | --------------------------------------------- | --------------------------------------- |
-| postgres_image                                | Path of the image to pull                     | quay.io/sclorg/postgresql-15-c9s:latest |
-| postgres_init_container_resource_requirements | Database init container resource requirements | requests: {cpu: 10m, memory: 64Mi}      |
-| postgres_resource_requirements                | PostgreSQL container resource requirements    | requests: {cpu: 10m, memory: 64Mi}      |
-| postgres_storage_requirements                 | PostgreSQL container storage requirements     | requests: {storage: 8Gi}                |
-| postgres_storage_class                        | PostgreSQL PV storage class                   | Empty string                            |
-| postgres_data_path                            | PostgreSQL data path                          | `/var/lib/postgresql/data/pgdata`       |
-| postgres_priority_class                       | Priority class used for PostgreSQL pod        | Empty string                            |
+| Name                                          | Description                                                     | Default                                 |
+| --------------------------------------------- | --------------------------------------------------------------- | --------------------------------------- |
+| postgres_image                                | Path of the image to pull                                       | quay.io/sclorg/postgresql-15-c9s        |
+| postgres_image_version                        | Image version to pull                                           | latest                                  |
+| postgres_resource_requirements                | PostgreSQL container (and initContainer) resource requirements  | requests: {cpu: 10m, memory: 64Mi}      |
+| postgres_storage_requirements                 | PostgreSQL container storage requirements                       | requests: {storage: 8Gi}                |
+| postgres_storage_class                        | PostgreSQL PV storage class                                     | Empty string                            |
+| postgres_priority_class                       | Priority class used for PostgreSQL pod                          | Empty string                            |
 
 Example of customization could be:
 
@@ -91,3 +90,30 @@ spec:
 ```
 
 **Note**: If `postgres_storage_class` is not defined, PostgreSQL will store it's data on a volume using the default storage class for your cluster.
+
+#### Note about overriding the postgres image
+
+We recommend you use the default image sclorg image. If you are coming from a deployment using the old postgres image from dockerhub (postgres:13), upgrading from awx-operator version 2.12.2 and below to 2.15.0+ will handle migrating your data to the new postgresql image (postgresql-15-c9s).
+
+You can no longer configure a custom `postgres_data_path` because it is hardcoded in the quay.io/sclorg/postgresql-15-c9s image.
+
+If you override the postgres image to use a custom postgres image like postgres:15 for example, the default data directory path may be different. These images cannot be used interchangeably.
+
+#### Initialize Postgres data volume
+
+When using a hostPath backed PVC and some other storage classes like longhorn storagfe, the postgres data directory needs to be accessible by the user in the postgres pod (UID 26).
+
+To initialize this directory with the correct permissions, configure the following setting, which will use an init container to set the permissions in the postgres volume.
+
+```yaml
+spec:
+  postgres_data_volume_init: true
+```
+
+Should you need to modify the init container commands, there is an example below.
+
+```yaml
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+```

molecule/default/molecule.yml

@@ -21,6 +21,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         default_awx_version: "{{ lookup('url', 'https://api.github.com/repos/ansible/awx/releases/latest') | from_json | json_query('tag_name') }}"
@@ -30,6 +31,8 @@ provisioner:
         operator_image: ${OPERATOR_IMAGE:-""}
         operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"}
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"}
 verifier:

molecule/default/tasks/apply_awx_spec.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create or update the awx.ansible.com/v1alpha1.AWX
+- name: Create or update the awx.ansible.com/v1beta1.AWX
   k8s:
     state: present
     namespace: '{{ namespace }}'

molecule/default/tasks/awx_test.yml

@@ -1,98 +1,8 @@
 ---
 - include_tasks: apply_awx_spec.yml
 
-- name: Obtain generated admin password
-  k8s_info:
-    namespace: '{{ namespace }}'
-    kind: Secret
-    name: example-awx-admin-password
-  register: admin_pw_secret
-
-- block:
-    - name: Get web pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-web
-      register: awx_web_pod
-      when: not awx_version
-
-    - name: Get task pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-task
-      register: awx_task_pod
-      when: not awx_version
-
-    - name: Extract tags from images from web pod
-      set_fact:
-        web_image_tags: |
-          {{ awx_web_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - name: Extract tags from images from task pod
-      set_fact:
-        task_image_tags: |
-          {{ awx_task_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - fail:
-        msg: |
-          It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
-          This is an environment variable that is set via build arg when releasing awx-operator.
-      when:
-        - not awx_version
-        - default_awx_version not in web_image_tags
-        - default_awx_version not in task_image_tags
-
-    - name: Launch Demo Job Template
-      awx.awx.job_launch:
-        name: Demo Job Template
-        wait: yes
-        validate_certs: no
-        controller_host: localhost/awx/
-        controller_username: admin
-        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-  rescue:
-    - name: Get list of project updates and jobs
-      uri:
-        url: "http://localhost/awx/api/v2/{{ resource }}/"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      register: job_lists
-      loop:
-        - project_updates
-        - jobs
-      loop_control:
-        loop_var: resource
-
-    - name: Get all job and project details
-      uri:
-        url: "http://localhost{{ endpoint }}"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      loop: |
-        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
-      loop_control:
-        loop_var: endpoint
-
-    - name: Re-emit failure
-      vars:
-        failed_task:
-          result: '{{ ansible_failed_result }}'
-      fail:
-        msg: '{{ failed_task }}'
-
-- block:
+- name: Validate AWX deployment
+  block:
     - name: Look up details for this AWX instance
       k8s_info:
         namespace: "{{ namespace }}"
@@ -117,65 +27,90 @@
           - app.kubernetes.io/name = example-awx-task
       register: awx_task_pod
 
-    - name: Extract additional_labels from AWX spec
-      set_fact:
-        awx_additional_labels: >-
-          {{ this_awx.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
+    - name: Validate DEFAULT_AWX_VERSION
+      block:
+      - name: Extract tags from images from web pod
+        set_fact:
+          web_image_tags: |
+            {{ awx_web_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - name: Extract tags from images from task pod
+        set_fact:
+          task_image_tags: |
+            {{ awx_task_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - fail:
+          msg: |
+            It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
+            This is an environment variable that is set via build arg when releasing awx-operator.
+        when:
+          - default_awx_version not in web_image_tags
+          - default_awx_version not in task_image_tags
+      when: not awx_version
 
-    - name: Extract additional_labels from AWX web Pod
-      set_fact:
-        awx_web_pod_additional_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
-
-    - name: Extract additional_labels from AWX task Pod
-      set_fact:
-        awx_task_pod_additional_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
+    - name: Validate additional_labels
+      block:
+      - name: Extract additional_labels from AWX spec
+        set_fact:
+          awx_additional_labels: >-
+            {{ this_awx.resources[0].metadata.labels
               | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
               | list
-          }}
+            }}
 
-    - name: Assert AWX web Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX web Pod
+        set_fact:
+          awx_web_pod_additional_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+              | list
+            }}
 
-    - name: Assert AWX task Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX task Pod
+        set_fact:
+          awx_task_pod_additional_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+                | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+                | list
+            }}
 
-    - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_web_pod_extra_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Assert AWX web Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_additional_labels == awx_additional_labels
 
-    - name: AWX web Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_extra_labels == []
+      - name: Assert AWX task Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_additional_labels == awx_additional_labels
 
-    - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_task_pod_extra_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_web_pod_extra_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
 
-    - name: AWX task Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_extra_labels == []
+      - name: AWX web Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_extra_labels == []
+
+      - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_task_pod_extra_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
+
+      - name: AWX task Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_extra_labels == []
 
   rescue:
     - name: Re-emit failure
@@ -184,3 +119,76 @@
           result: '{{ ansible_failed_result }}'
       fail:
         msg: '{{ failed_task }}'
+
+- name: Obtain generated admin password
+  k8s_info:
+    namespace: '{{ namespace }}'
+    kind: Secret
+    name: example-awx-admin-password
+  register: admin_pw_secret
+
+- name: Validate demo job launch
+  block:
+    - name: Launch Demo Job Template
+      awx.awx.job_launch:
+        name: Demo Job Template
+        wait: yes
+        validate_certs: no
+        controller_host: localhost/awx/
+        controller_username: admin
+        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+
+  rescue:
+    - name: Get list of project updates and jobs
+      uri:
+        url: "http://localhost/awx/api/v2/{{ resource }}/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: job_lists
+      loop:
+        - project_updates
+        - jobs
+      loop_control:
+        loop_var: resource
+
+    - name: Store job_lists debug output
+      copy:
+        content: "{{ job_lists | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_lists.json"
+      when: store_debug_output | default(false)
+
+    - name: Get all job and project details
+      uri:
+        url: "http://localhost{{ endpoint }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
+      loop_control:
+        loop_var: endpoint
+      register: job_details
+
+    - name: Store job_details debug output
+      copy:
+        content: "{{ job_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_details.json"
+      when: store_debug_output | default(false)
+
+    ## TODO: figure out why this doesn't work
+    # - name: Store debug outputs
+    #   copy:
+    #     content: '{{ item }}'
+    #     dest: "{{ debug_output_dir }}/{{ item }}.json"
+    #   loop:
+    #     - job_lists
+    #     - job_details
+    #   when: store_debug_output | default(false)
+
+    - name: Re-emit failure
+      vars:
+        failed_task:
+          result: '{{ ansible_failed_result }}'
+      fail:
+        msg: '{{ failed_task }}'

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -13,6 +13,12 @@ spec:
 {% endif %}
 {% if awx_version %}
   image_version: {{ awx_version }}
+{% endif %}
+{% if awx_ee_image %}
+  control_plane_ee_image: {{ awx_ee_image }}
+  ee_images:
+  - image: {{ awx_ee_image }}
+    name: AWX EE
 {% endif %}
   ingress_type: ingress
   ingress_path: /awx
@@ -32,11 +38,10 @@ spec:
       memory: 16M
   no_log: false
   postgres_resource_requirements: {}
-  postgres_init_container_resource_requirements: {}
   redis_resource_requirements: {}
   additional_labels:
   - my/team
   - my/service
 {% if additional_fields is defined %}
   {{ additional_fields | to_nice_yaml | indent(2) }}
-{% endif %}
+{% endif %}

molecule/default/utils/output_all_container_logs_for_pod.yml

@@ -0,0 +1,15 @@
+---
+- name: Get all container log in pod
+  kubernetes.core.k8s_log:
+    namespace: '{{ namespace }}'
+    name: '{{ item.metadata.name }}'
+    all_containers: true
+  register: all_container_logs
+
+- name: Store logs in file
+  ansible.builtin.copy:
+    content: "{{ all_container_logs.log_lines | join('\n') }}"
+    dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log'
+
+# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs
+# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported

molecule/default/utils/output_k8s_resources.yml

@@ -0,0 +1,29 @@
+---
+- name: Retrieve relevant k8s resources
+  kubernetes.core.k8s_info:
+    api_version: '{{ item.api_version }}'
+    kind: '{{ item.kind }}'
+    namespace: '{{ namespace }}'
+  loop:
+    - api_version: v1
+      kind: Pod
+    - api_version: apps/v1
+      kind: Deployment
+    - api_version: v1
+      kind: Secret
+    - api_version: v1
+      kind: ConfigMap
+    - api_version: "awx.ansible.com/v1beta1"
+      kind: AWX
+  register: debug_resources
+
+- name: debug print item.kind and item.metadata.name
+  debug:
+    msg: '{{ item.kind }}-{{ item.metadata.name }}'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"
+
+- name: Output gathered resource to files
+  ansible.builtin.copy:
+    content: '{{ item | to_nice_json }}'
+    dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"

molecule/default/verify.yml

@@ -10,53 +10,41 @@
     ctrl_label: control-plane=controller-manager
 
   tasks:
-    - block:
+    - name: Perform awx tests
+      block:
         - name: Import all test files from tasks/
-          include_tasks: '{{ item }}'
+          ansible.builtin.include_tasks: '{{ item }}'
           with_fileglob:
             - tasks/awx_test.yml
             - tasks/awx_replicas_test.yml
           tags:
             - always
       rescue:
-        - name: Retrieve relevant resources
-          k8s_info:
-            api_version: '{{ item.api_version }}'
-            kind: '{{ item.kind }}'
+        - name: Create debug output directory
+          ansible.builtin.file:
+            path: '{{ debug_output_dir }}'
+            state: directory
+          tags:
+            - always
+
+        - name: Gather and output K8s resources
+          ansible.builtin.include_tasks: utils/output_k8s_resources.yml
+          tags:
+            - always
+
+        - name: Get all pods
+          kubernetes.core.k8s_info:
+            api_version: v1
+            kind: Pod
             namespace: '{{ namespace }}'
-          loop:
-            - api_version: v1
-              kind: Pod
-            - api_version: apps/v1
-              kind: Deployment
-            - api_version: v1
-              kind: Secret
-            - api_version: v1
-              kind: ConfigMap
-          register: debug_resources
+          register: all_pods
           tags:
             - always
 
-        - name: Retrieve Pod logs
-          k8s_log:
-            name: '{{ item.metadata.name }}'
-            namespace: '{{ namespace }}'
-            container: awx-manager
-          loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}"
-          register: debug_logs
-          tags:
-            - always
-
-        - name: Output gathered resources
-          debug:
-            var: debug_resources
-          tags:
-            - always
-
-        - name: Output gathered logs
-          debug:
-            var: item.log_lines
-          loop: '{{ debug_logs.results }}'
+        - name: Get all container logs for all pods
+          ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml
+          loop: '{{ all_pods.resources }}'
+          ignore_errors: yes
           tags:
             - always
 
@@ -64,7 +52,7 @@
           vars:
             failed_task:
               result: '{{ ansible_failed_result }}'
-          fail:
+          ansible.builtin.fail:
             msg: '{{ failed_task }}'
           tags:
             - always

molecule/kind/molecule.yml

@@ -23,6 +23,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         ansible_python_interpreter: '{{ ansible_playbook_python }}'
@@ -34,6 +35,8 @@ provisioner:
         operator_pull_policy: "Never"
         kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig
     KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig

noxfile.py

@@ -0,0 +1,20 @@
+import nox
+
+
+@nox.session
+def build(session: nox.Session):
+    """
+    Build the AWX Operator docsite.
+    """
+    session.install(
+        "-r",
+        "docs/requirements.in",
+        "-c",
+        "docs/requirements.txt",
+    )
+    session.run(
+        "mkdocs",
+        "build",
+        "--strict",
+        *session.posargs,
+    )

roles/backup/tasks/secrets.yml

@@ -20,9 +20,7 @@
 
 - name: Dump ingress tls secret names from awx spec and data into file
   include_tasks: dump_ingress_tls_secrets.yml
-  with_items:
-    - "{{ awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list }}"
-  when: awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list | length
+  with_items: "{{ awx_spec.spec['ingress_hosts'] | default([]) | selectattr('tls_secret', 'defined') | map(attribute='tls_secret') | list }}"
 
 - name: Dump receptor secret names and data into file
   include_tasks: dump_receptor_secrets.yml

roles/installer/defaults/main.yml

@@ -296,6 +296,11 @@ replicas: 1
 web_replicas: ''
 task_replicas: ''
 
+web_liveness_period: 0
+web_readiness_period: 0
+task_liveness_period: 0
+task_readiness_period: 0
+
 task_args:
   - /usr/bin/launch_awx_task.sh
 task_command: []
@@ -395,13 +400,9 @@ postgres_resource_requirements:
   requests:
     cpu: 10m
     memory: 64Mi
-postgres_init_container_resource_requirements:
-  requests:
-    cpu: 10m
-    memory: 64Mi
+
 # Assign a preexisting priority class to the postgres pod
 postgres_priority_class: ''
-postgres_data_path: '/var/lib/pgsql/data/pgdata'
 
 # Persistence to the AWX project data folder
 # Whether or not the /var/lib/projects directory will be persistent
@@ -413,6 +414,11 @@ projects_existing_claim: ''
 # Define postgres configuration arguments to use
 postgres_extra_args: ''
 
+postgres_data_volume_init: false
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+
 # Configure postgres connection keepalive
 postgres_keepalives: true
 postgres_keepalives_idle: 5
@@ -488,9 +494,11 @@ nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
 # metrics-utility (github.com/ansible/metrics-utility)
 _metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
 _metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
+_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
 _metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
 _metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
 _metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
+_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}"
 _metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
 _metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
 _metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"

roles/installer/tasks/database_configuration.yml

@@ -129,7 +129,7 @@
       set_fact:
         sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
           sort(attribute='metadata.name') |
-          reverse }}"
+          reverse | list }}"
       when: filtered_old_postgres_pods | length
 
 
@@ -162,8 +162,8 @@
         pod: "{{ old_postgres_pod['metadata']['name'] }}"
         command: |
           bash -c """
-          if [ -f "{{ postgres_data_path }}/PG_VERSION" ]; then
-            cat "{{ postgres_data_path }}/PG_VERSION"
+          if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then
+            cat "{{ _postgres_data_path }}/PG_VERSION"
           elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then
             cat '/var/lib/postgresql/data/pgdata/PG_VERSION'
           fi

roles/installer/tasks/enable_metrics_utility.yml

@@ -1,23 +1,42 @@
 ---
-# Check to make sure provided pvc exists, error loudly if not.  Otherwise, the management pod will just stay in pending state forever.
-- name: Check provided PVC claim exists
-  kubernetes.core.k8s_info:
-    name: "{{ _metrics_utility_pvc_claim }}"
-    kind: PersistentVolumeClaim
-    namespace: "{{ ansible_operator_meta.namespace }}"
-  when:
-    - _metrics_utility_pvc_claim | length
+- name: Setup PVC if using directory ship target
+  block:
 
-- name: Create PVC for metrics-utility
-  kubernetes.core.k8s:
-    kind: PersistentVolumeClaim
-    definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+    # Check to make sure provided pvc exists
+    - name: Check provided PVC claim exists
+      kubernetes.core.k8s_info:
+        name: "{{ _metrics_utility_pvc_claim }}"
+        kind: PersistentVolumeClaim
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      when:
+        - _metrics_utility_pvc_claim | length
 
-- name: Create Kubernetes CronJobs for metrics-utility
+    - name: Create PVC for metrics-utility
+      kubernetes.core.k8s:
+        kind: PersistentVolumeClaim
+        definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+
+  when: _metrics_utility_ship_target == "directory"
+
+- name: Create default metrics-utility Kubernetes CronJobs
   kubernetes.core.k8s:
-    definition: "{{ lookup('template', item) }}"
+    definition: "{{ lookup('template', item.template) }}"
     apply: true
     wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
   loop:
-    - cronjobs/metrics-utility-gather.yaml.j2
-    - cronjobs/metrics-utility-report.yaml.j2
+    - {name: 'metrics-utility-gather', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+    - {name: 'metrics-utility-report', template: 'cronjobs/metrics-utility-report.yaml.j2'}
+
+- name: Create metrics-utility Kubernetes CronJob for Red Hat Hybrid Cloud Console
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', item.template) }}"
+    apply: true
+    wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
+    metrics_utility_ship_target: crc  # TODO - Update to console when changed
+  loop:
+    - {name: 'metrics-utility-gather-console', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+  when: _metrics_utility_console_enabled

roles/installer/tasks/resources_configuration.yml

@@ -266,6 +266,11 @@
         field_selectors:
           - status.phase=Running
       register: _new_pod
+      until:
+        - "_new_pod['resources'] | length"
+        - "_new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | length"
+      retries: 60
+      delay: 5
 
     - name: Update new resource pod as a variable.
       set_fact:

roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2

@@ -2,10 +2,10 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+  name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
     {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
@@ -16,7 +16,7 @@ spec:
       template:
         metadata:
           labels:
-            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
             {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
             {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
         spec:
@@ -30,7 +30,7 @@ spec:
 {% endfor %}
 {% endif %}
           containers:
-          - name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+          - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
             image: "{{ _metrics_utility_image }}"
             imagePullPolicy: "{{ image_pull_policy }}"
             resources:
@@ -41,9 +41,16 @@ spec:
               - /bin/sh
               - -c
               - metrics-utility gather_automation_controller_billing_data --ship --until=10m
+            env:
+            - name: METRICS_UTILITY_SHIP_TARGET
+              value: "{{ _metrics_utility_ship_target }}"
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
+{% if _metrics_utility_secret is defined %}
+            - secretRef:
+                name: {{ _metrics_utility_secret }}
+{% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
               mountPath: /metrics-utility

roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2

@@ -2,10 +2,10 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: {{ ansible_operator_meta.name }}-metrics-utility-report
+  name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
     {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
@@ -16,7 +16,7 @@ spec:
       template:
         metadata:
           labels:
-            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
             {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
             {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
         spec:
@@ -30,7 +30,7 @@ spec:
 {% endfor %}
 {% endif %}
           containers:
-          - name: {{ ansible_operator_meta.name }}-metrics-utility-report
+          - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
             image: "{{ _metrics_utility_image }}"
             imagePullPolicy: "{{ image_pull_policy }}"
             resources:
@@ -44,6 +44,10 @@ spec:
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
+{% if _metrics_utility_secret is defined %}
+            - secretRef:
+                name: {{ _metrics_utility_secret }}
+{% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
               mountPath: /metrics-utility

roles/installer/templates/deployments/web.yaml.j2

@@ -178,11 +178,10 @@ spec:
 {% endif %}
 {% if web_readiness_period|int > 0 %}
           readinessProbe:
-            exec:
-              httpGet:
-                path: /api/v2/ping/
-                scheme: HTTP
-                port: 8052
+            httpGet:
+              path: /api/v2/ping/
+              scheme: HTTP
+              port: 8052
             initialDelaySeconds: {{ web_readiness_initial_delay }}
             periodSeconds: {{ web_readiness_period }}
             failureThreshold: {{ web_readiness_failure_threshold }}

roles/installer/templates/statefulsets/postgres.yaml.j2

@@ -46,6 +46,27 @@ spec:
 {% endif %}
 {% if postgres_priority_class is defined %}
       priorityClassName: '{{ postgres_priority_class }}'
+{% endif %}
+{% if postgres_data_volume_init and not is_openshift %}
+      initContainers:
+        - name: init
+          image: '{{ _postgres_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          securityContext:
+            runAsUser: 0
+          command:
+            - /bin/sh
+            - -c
+            - |
+              {{ postgres_init_container_commands | indent(width=14) }}
+          resources: {{ postgres_init_container_resource_requirements | default(postgres_resource_requirements) }}
+          volumeMounts:
+            - name: postgres-{{ supported_pg_version }}
+              mountPath: '{{ _postgres_data_path | dirname }}'
+              subPath: '{{ _postgres_data_path | dirname | basename }}'
+{% if postgres_extra_volume_mounts %}
+            {{ postgres_extra_volume_mounts | indent(width=12, first=True) }}
+{% endif %}
 {% endif %}
       containers:
         - image: '{{ _postgres_image }}'
@@ -94,7 +115,7 @@ spec:
                   name: '{{ __postgres_configuration_secret }}'
                   key: password
             - name: PGDATA
-              value: '{{ postgres_data_path }}'
+              value: '{{ _postgres_data_path }}'
             - name: POSTGRES_INITDB_ARGS
               value: '{{ postgres_initdb_args }}'
             - name: POSTGRES_HOST_AUTH_METHOD
@@ -111,9 +132,9 @@ spec:
               name: postgres-{{ supported_pg_version }}
           volumeMounts:
             - name: postgres-{{ supported_pg_version }}
-              mountPath: '{{ postgres_data_path | dirname }}'
-              subPath: '{{ postgres_data_path | dirname | basename }}'
-{% if postgres_extra_volume_mounts -%}
+              mountPath: '{{ _postgres_data_path | dirname }}'
+              subPath: '{{ _postgres_data_path | dirname | basename }}'
+{% if postgres_extra_volume_mounts %}
             {{ postgres_extra_volume_mounts | indent(width=12, first=True) }}
 {% endif %}
           resources: {{ postgres_resource_requirements }}

roles/installer/vars/main.yml

@@ -7,3 +7,4 @@ projects_existing_claim: ''
 supported_pg_version: 15
 _previous_upgraded_pg_version: 0
 old_postgres_pod: []
+_postgres_data_path: '/var/lib/pgsql/data/userdata'

roles/mesh_ingress/templates/ingress.yml.j2

@@ -70,7 +70,7 @@ metadata:
   name: {{ ansible_operator_meta.name }}
   namespace: {{ ansible_operator_meta.namespace }}
 spec:
-{% if external_hostname %}
+{% if external_hostname is defined %}
   host: {{ external_hostname }}
 {% endif %}
   port: