Wait for instance ready in molecule test (#1901)

Sometimes a job is launched through the web api
before the instance is in a ready state. This throws
a 500 internal server error, causing CI to fail.

Adds a task to query the instances endpoint
and check that at least one control node is
in a ready state.

Signed-off-by: Seth Foster <fosterbseth@gmail.com>

.github/workflows/ci.yaml

@@ -17,6 +17,7 @@ jobs:
           - -t replicas
     env:
       DOCKER_API_VERSION: "1.41"
+      DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
     steps:
       - uses: actions/checkout@v3
 
@@ -37,10 +38,18 @@ jobs:
           MOLECULE_VERBOSITY: 3
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          STORE_DEBUG_OUTPUT: true
         run: |
           sudo rm -f $(which kustomize)
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }}
+
+      - name: Upload artifacts for failed tests if Run Molecule fails
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: awx_operator_molecule_test
+          path: ${{ env.DEBUG_OUTPUT_DIR }}
   helm:
     runs-on: ubuntu-latest
     name: helm

.github/workflows/reusable-nox.yml

@@ -18,7 +18,7 @@ jobs:
         - name: Check out repo
           uses: actions/checkout@v4
         - name: Setup nox
-          uses: wntrblm/nox@2024.03.02
+          uses: wntrblm/nox@2024.04.15
           with:
               python-versions: "${{ matrix.python-versions }}"
         - name: "Run nox -s ${{ matrix.session }}"

.helm/starter/templates/postgres-config.yaml

@@ -12,6 +12,7 @@ stringData:
   username: {{ .username }}
   password: {{ .password }}
   sslmode: {{ .sslmode }}
+  target_session_attrs: {{ .target_session_attrs | default "any" }}
   type: {{ .type }}
 type: Opaque
 {{- end }}

config/crd/bases/awx.ansible.com_awxmeshingresses.yaml

@@ -75,6 +75,381 @@ spec:
               ingress_controller:
                 description: Special configuration for specific Ingress Controllers
                 type: string
+              node_selector:
+                description: Assign the Mesh Ingress Pod to the specified node.
+                type: string
+              tolerations:
+                description: Scheduling tolerations for the Mesh Ingress instance.
+                type: string
+              topology_spread_constraints:
+                description: Topology spread constraints for the Mesh Ingress instance.
+                type: string
+              affinity:
+                description: Scheduling constraints to apply to the Pod definition
+                properties:
+                  nodeAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            preference:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - preference
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        properties:
+                          nodeSelectorTerms:
+                            items:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            type: array
+                        required:
+                        - nodeSelectorTerms
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                  podAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaceSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaceSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                  podAntiAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaceSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaceSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                type: object
           status:
             description: Status defines the observed state of AWXMeshIngress
             type: object

config/crd/bases/awx.ansible.com_awxrestores.yaml

@@ -94,6 +94,11 @@ spec:
               postgres_image_version:
                 description: PostgreSQL container image version to use
                 type: string
+              spec_overrides:
+                description: Overrides for the AWX spec
+                # type: string
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
               image_pull_policy:
                 description: The image pull policy
                 type: string

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -144,6 +144,9 @@ spec:
               ingress_controller:
                 description: Special configuration for specific Ingress Controllers
                 type: string
+              api_urlpattern_prefix:
+                description: An optional configuration to add a prefix in the API URL path
+                type: string
               loadbalancer_protocol:
                 description: Protocol to use for the loadbalancer
                 type: string
@@ -1571,10 +1574,18 @@ spec:
                 description: Number of web instance replicas
                 type: integer
                 format: int32
+              web_manage_replicas:
+                description: Enables operator control of replicas count for the web deployment when set to 'true'
+                type: boolean
+                default: true
               task_replicas:
                 description: Number of task instance replicas
                 type: integer
                 format: int32
+              task_manage_replicas:
+                description: Enables operator control of replicas count for the task deployment when set to 'true'
+                type: boolean
+                default: true
               web_liveness_initial_delay:
                 description: Initial delay before starting liveness checks on web pod
                 type: integer
@@ -1901,6 +1912,28 @@ spec:
                       x-kubernetes-preserve-unknown-fields: true
                   type: object
                 type: array
+              extra_settings_files:
+                description: Extra ConfigMaps or Secrets of settings files to specify for AWX
+                properties:
+                  configmaps:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                  secrets:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                type: object
               no_log:
                 description: Configure no_log for no_log tasks
                 type: boolean
@@ -1940,26 +1973,36 @@ spec:
                 description: Metrics-Utility Image PullPolicy
                 type: string
               metrics_utility_configmap:
-                description: Metrics-Utlity ConfigMap
+                description: Metrics-Utility ConfigMap
+                type: string
+              metrics_utility_secret:
+                description: Metrics-Utility Secret
                 type: string
               metrics_utility_cronjob_gather_schedule:
-                description: Metrics-Utlity Gather Data CronJob Schedule
+                description: Metrics-Utility Gather Data CronJob Schedule
                 type: string
                 default: '@hourly'
               metrics_utility_cronjob_report_schedule:
-                description: Metrics-Utlity Report CronJob Schedule
+                description: Metrics-Utility Report CronJob Schedule
                 type: string
                 default: '@monthly'
+              metrics_utility_ship_target:
+                description: Metrics-Utility Ship Target
+                type: string
               metrics_utility_pvc_claim:
-                description: Metrics-Utlity PVC Claim
+                description: Metrics-Utility PVC Claim
                 type: string
               metrics_utility_pvc_claim_size:
-                description: Metrics-Utlity PVC Claim Size
+                description: Metrics-Utility PVC Claim Size
                 type: string
                 default: 5Gi
               metrics_utility_pvc_claim_storage_class:
-                description: Metrics-Utlity PVC Claim Storage Class
+                description: Metrics-Utility PVC Claim Storage Class
                 type: string
+              metrics_utility_console_enabled:
+                description: Enable metrics utility shipping to Red Hat Hybrid Cloud Console
+                type: boolean
+                default: false
             type: object
           status:
             properties:

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -58,6 +58,31 @@ spec:
         path: ingress_controller
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Node Selector
+        path: node_selector
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Tolerations
+        path: tolerations
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Topology Spread Constraints
+        path: topology_spread_constraints
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Affinity
+        path: affinity
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Optional API URLPATTERN Prefix
+        path: api_urlpattern_prefix
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
       - displayName: Image Pull Secrets
         path: image_pull_secrets
         x-descriptors:
@@ -213,6 +238,10 @@ spec:
         path: postgres_image_version
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: AWX Spec Overrides
+        path: spec_overrides
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Image Pull Policy
         path: image_pull_policy
         x-descriptors:
@@ -957,6 +986,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Extra Settings Files
+        path: extra_settings_files
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: No Log Configuration
         path: no_log
         x-descriptors:
@@ -1039,7 +1073,7 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Image Version
+      - displayName: Metrics-Utility Image Version
         path: metrics_utility_image_version
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
@@ -1051,42 +1085,60 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:imagePullPolicy
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity ConfigMap
+      - displayName: Metrics-Utility ConfigMap
         path: metrics_utility_configmap
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:ConfigMap
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Gather Data CronJob Schedule
+      - displayName: Metrics-Utility Secret
+        path: metrics_utility_secret
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:Secret
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Gather Data CronJob Schedule
         path: metrics_utility_cronjob_gather_schedule
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity Report CronJob Schedule
+      - displayName: Metrics-Utility Report CronJob Schedule
         path: metrics_utility_cronjob_report_schedule
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim
+      - displayName: Metrics-Utility Ship Target
+        path: metrics_utility_ship_target
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility PVC Claim
         path: metrics_utility_pvc_claim
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim Size
+      - displayName: Metrics-Utility PVC Claim Size
         path: metrics_utility_pvc_claim_size
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
-      - displayName: Metrics-Utlity PVC Claim Storage Class
+      - displayName: Metrics-Utility PVC Claim Storage Class
         path: metrics_utility_pvc_claim_storage_class
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Enabled Shipping to Red Hat Hybrid Cloud Console
+        path: metrics_utility_console_enabled
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
       statusDescriptors:
       - description: Route to access the instance deployed
         displayName: URL

docs/README.md

@@ -2,9 +2,15 @@
 
 To build the AWX Operator docs locally:
 
-1. Clone the AWX operator repository. 
-2. From the root directory:
-    a.  pip install --user -r docs/requirements.txt
-    b.  mkdocs build
+1. Clone the AWX operator repository.
+1. Preferrably, create a virtual environment for installing the dependencies.  
+   a. `python3 -m venv venv`  
+   b. `source venv/bin/activate`
+1. From the root directory:  
+   a. `pip install -r docs/requirements.txt`  
+   b. `mkdocs build`
+1. View the docs in your browser:  
+   a. `mkdocs serve`  
+   b. Open your browser and navigate to `http://127.0.0.1:8000/`
 
-This will create a new directory called `site/` in the root of your clone containing the index.html and static files. To view the docs in your browser, navigate there in your file explorer and double-click on the `index.html` file. This should open the docs site in your browser.
+This will create a new directory called `site/` in the root of your clone containing the index.html and static files.

docs/requirements.txt

@@ -50,6 +50,7 @@ idna==3.6
 jinja2==3.1.3
     # via
     #   mkdocs
+    #   mkdocs-macros-plugin
     #   mkdocs-material
     #   mkdocstrings
 jsmin==3.0.1
@@ -88,18 +89,21 @@ mkdocs==1.5.3
     #   mkdocs-autorefs
     #   mkdocs-gen-files
     #   mkdocs-htmlproofer-plugin
+    #   mkdocs-macros-plugin
     #   mkdocs-material
     #   mkdocs-minify-plugin
     #   mkdocs-monorepo-plugin
     #   mkdocstrings
-mkdocs-ansible==24.2.1
-    # via -r docs/requirements.in
+mkdocs-ansible==24.3.1
+    # via -r requirements.in
 mkdocs-autorefs==0.5.0
     # via mkdocstrings
 mkdocs-gen-files==0.5.0
     # via mkdocs-ansible
 mkdocs-htmlproofer-plugin==1.0.0
     # via mkdocs-ansible
+mkdocs-macros-plugin==1.0.5
+    # via mkdocs-ansible
 mkdocs-material==9.2.6
     # via mkdocs-ansible
 mkdocs-material-extensions==1.3.1
@@ -145,12 +149,15 @@ pymdown-extensions==10.0.1
 pyquery==2.0.0
     # via readtime
 python-dateutil==2.8.2
-    # via ghp-import
+    # via
+    #   ghp-import
+    #   mkdocs-macros-plugin
 python-slugify==8.0.4
     # via mkdocs-monorepo-plugin
 pyyaml==6.0.1
     # via
     #   mkdocs
+    #   mkdocs-macros-plugin
     #   pymdown-extensions
     #   pyyaml-env-tag
 pyyaml-env-tag==0.1
@@ -168,6 +175,8 @@ six==1.16.0
     # via python-dateutil
 soupsieve==2.5
     # via beautifulsoup4
+termcolor==2.4.0
+    # via mkdocs-macros-plugin
 text-unidecode==1.3
     # via python-slugify
 tinycss2==1.2.1

docs/upgrade/upgrading.md

@@ -1,6 +1,7 @@
 ### Upgrading
 
-To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX.  To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command
+To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX. To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command
+
 ```shell
 AWX_OPERATOR_VERSION=2.8.0
 docker run --entrypoint="" quay.io/ansible/awx-operator:$AWX_OPERATOR_VERSION bash -c "env | grep DEFAULT_AWX_VERSION"
@@ -12,26 +13,23 @@ Apply the awx-operator.yml for that release to upgrade the operator, and in turn
 
 The first part of any upgrade should be a backup. Note, there are secrets in the pod which work in conjunction with the database. Having just a database backup without the required secrets will not be sufficient for recovering from an issue when upgrading to a new version. See the [backup role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/backup) for information on how to backup your database and secrets.
 
-In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). *Before Restoring from a backup*, be sure to:
-* delete the old existing AWX CR
-* delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
+In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). _Before Restoring from a backup_, be sure to:
+
+- delete the old existing AWX CR
+- delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
 
 **Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well.
 
-
 #### PostgreSQL Upgrade Considerations
 
 If there is a PostgreSQL major version upgrade, after the data directory on the PVC is migrated to the new version, the old PVC is kept by default.
-This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically
-after a successful upgrade by setting the following variable on the AWX spec. 
-
+This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically after a successful upgrade by setting the following variable on the AWX spec.
 
 ```yaml
-  spec:
-    postgres_keep_pvc_after_upgrade: False
+spec:
+  postgres_keep_pvc_after_upgrade: False
 ```
 
-
 #### v0.14.0
 
 ##### Cluster-scope to Namespace-scope considerations

docs/user-guide/advanced-configuration/assigning-awx-pods-to-specific-nodes.md

@@ -88,3 +88,8 @@ spec:
               - S2
           topologyKey: topology.kubernetes.io/zone
 ```
+
+#### Special Note on DB-Migration Job Scheduling
+
+For the **db-migration job**, which applies database migrations at cluster startup, you can specify scheduling settings using the `task_*` configurations such as `task_node_selector`, `task_tolerations`, etc.  
+If these task-specific settings are not defined, the job will automatically use the global AWX configurations like `node_selector` and `tolerations`.

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -1,4 +1,4 @@
-#### Custom Volume and Volume Mount Options
+# Custom Volume and Volume Mount Options
 
 In a scenario where custom volumes and volume mounts are required to either overwrite defaults or mount configuration files.
 
@@ -12,8 +12,8 @@ In a scenario where custom volumes and volume mounts are required to either over
 | init_container_extra_volume_mounts | Specify volume mounts to be added to Init container      | ''      |
 | init_container_extra_commands      | Specify additional commands for Init container           | ''      |
 
-
-> :warning: The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
+!!! warning
+    The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
 
 Example configuration for ConfigMap
 
@@ -26,64 +26,50 @@ metadata:
   namespace: <target namespace>
 data:
   ansible.cfg: |
-     [defaults]
-     remote_tmp = /tmp
-     [ssh_connection]
-     ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
-  custom.py:  |
-      INSIGHTS_URL_BASE = "example.org"
-      AWX_CLEANUP_PATHS = True
+    [defaults]
+    remote_tmp = /tmp
+    [ssh_connection]
+    ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
 ```
+
 Example spec file for volumes and volume mounts
 
 ```yaml
 ---
-    spec:
-    ...
-      extra_volumes: |
-        - name: ansible-cfg
-          configMap:
-            defaultMode: 420
-            items:
-              - key: ansible.cfg
-                path: ansible.cfg
-            name: <resourcename>-extra-config
-        - name: custom-py
-          configMap:
-            defaultMode: 420
-            items:
-              - key: custom.py
-                path: custom.py
-            name: <resourcename>-extra-config
-        - name: shared-volume
-          persistentVolumeClaim:
-            claimName: my-external-volume-claim
+spec:
+  ...
+  extra_volumes: |
+    - name: ansible-cfg
+      configMap:
+        defaultMode: 420
+        items:
+          - key: ansible.cfg
+            path: ansible.cfg
+        name: <resourcename>-extra-config
+    - name: shared-volume
+      persistentVolumeClaim:
+        claimName: my-external-volume-claim
 
-      init_container_extra_volume_mounts: |
-        - name: shared-volume
-          mountPath: /shared
+  init_container_extra_volume_mounts: |
+    - name: shared-volume
+      mountPath: /shared
 
-      init_container_extra_commands: |
-        # set proper permissions (rwx) for the awx user
-        chmod 775 /shared
-        chgrp 1000 /shared
+  init_container_extra_commands: |
+    # set proper permissions (rwx) for the awx user
+    chmod 775 /shared
+    chgrp 1000 /shared
 
-      ee_extra_volume_mounts: |
-        - name: ansible-cfg
-          mountPath: /etc/ansible/ansible.cfg
-          subPath: ansible.cfg
-
-      task_extra_volume_mounts: |
-        - name: custom-py
-          mountPath: /etc/tower/conf.d/custom.py
-          subPath: custom.py
-        - name: shared-volume
-          mountPath: /shared
+  ee_extra_volume_mounts: |
+    - name: ansible-cfg
+      mountPath: /etc/ansible/ansible.cfg
+      subPath: ansible.cfg
 ```
 
-> :warning: **Volume and VolumeMount names cannot contain underscores(_)**
+!!! warning
+    **Volume and VolumeMount names cannot contain underscores(_)**
+
+## Custom UWSGI Configuration
 
-##### Custom UWSGI Configuration
 We allow the customization of two UWSGI parameters:
 
 * [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5)
@@ -103,7 +89,7 @@ requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related nginx
 configuration.
 
-##### Custom Nginx Configuration
+## Custom Nginx Configuration
 
 Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf.
 
@@ -124,26 +110,70 @@ may allow the web pods to handle more "bursty" request patterns if many
 requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related uwsgi
 configuration.
+
 * [worker_processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes) with `nginx_worker_processes` (default of 1)
 * [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) with `nginx_worker_cpu_affinity` (default "auto")
 * [worker_connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) with `nginx_worker_connections` (minimum of 1024)
 * [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size)
 
+## Custom Logos
 
-##### Custom Favicon
+You can use custom volume mounts to mount in your own logos to be displayed instead of the AWX logo.
+There are two different logos, one to be displayed on page headers, and one for the login screen.
 
-You can use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
-
-First, Create the configmap from a local favicon.ico file.
+First, create configmaps for the logos from local `logo-login.svg` and `logo-header.svg` files.
 
 ```bash
-$ oc create configmap favicon-configmap --from-file favicon.ico
+kubectl create configmap logo-login-configmap --from-file logo-login.svg
+kubectl create configmap logo-header-configmap --from-file logo-header.svg
 ```
 
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
 
 ```yaml
+---
 spec:
+  ...
+  extra_volumes: |
+    - name: logo-login
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-login.svg
+            path: logo-login.svg
+        name: logo-login-configmap
+    - name: logo-header
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-header.svg
+            path: logo-header.svg
+        name: logo-header-configmap
+  web_extra_volume_mounts: |
+    - name: logo-login
+      mountPath: /var/lib/awx/public/static/media/logo-login.svg
+      subPath: logo-login.svg
+    - name: logo-header
+      mountPath: /var/lib/awx/public/static/media/logo-header.svg
+      subPath: logo-header.svg
+```
+
+## Custom Favicon
+
+You can also use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
+
+First, create the configmap from a local `favicon.ico` file.
+
+```bash
+kubectl create configmap favicon-configmap --from-file favicon.ico
+```
+
+Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
+
+```yaml
+---
+spec:
+  ...
   extra_volumes: |
     - name: favicon
       configMap:
@@ -157,3 +187,7 @@ spec:
       mountPath: /var/lib/awx/public/static/media/favicon.ico
       subPath: favicon.ico
 ```
+
+## Custom AWX Configuration
+
+Refer to the [Extra Settings](./extra-settings.md) documentation for customizing the AWX configuration.

docs/user-guide/advanced-configuration/extra-settings.md

@@ -1,26 +1,119 @@
-#### Extra Settings
+# Extra Settings
 
-With`extra_settings`, you can pass multiple custom settings via the `awx-operator`. The parameter `extra_settings`  will be appended to the `/etc/tower/settings.py` and can be an alternative to the `extra_volumes` parameter.
+With `extra_settings` and `extra_settings_files`, you can pass multiple custom settings to AWX via the AWX Operator.
 
-| Name           | Description    | Default |
-| -------------- | -------------- | ------- |
-| extra_settings | Extra settings | ''      |
+!!! note
+    Parameters configured in `extra_settings` or `extra_settings_files` are set as read-only settings in AWX. As a result, they cannot be changed in the UI after deployment.
 
-**Note:** Parameters configured in `extra_settings` are set as read-only settings in AWX.  As a result, they cannot be changed in the UI after deployment. If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec.  
+    If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec (for `extra_settings`) or corresponding ConfigMap or Secret (for `extra_settings_files`). After updating ConfigMap or Secret, you need to restart the AWX pods to apply the changes.
+
+!!! note
+    If the same setting is set in both `extra_settings` and `extra_settings_files`, the setting in `extra_settings_files` will take precedence.
+
+## Add extra settings with `extra_settings`
+
+You can pass extra settings by specifying the pair of the setting name and value as the `extra_settings` parameter.
+
+The settings passed via `extra_settings` will be appended to the `/etc/tower/settings.py`.
+
+| Name           | Description    | Default   |
+| -------------- | -------------- | --------- |
+| extra_settings | Extra settings | `[]`      |
 
 Example configuration of `extra_settings` parameter
 
 ```yaml
-  spec:
-    extra_settings:
-      - setting: MAX_PAGE_SIZE
-        value: "500"
+spec:
+  extra_settings:
+    - setting: MAX_PAGE_SIZE
+      value: "500"
 
-      - setting: AUTH_LDAP_BIND_DN
-        value: "cn=admin,dc=example,dc=com"
+    - setting: AUTH_LDAP_BIND_DN
+      value: "cn=admin,dc=example,dc=com"
 
-      - setting: LOG_AGGREGATOR_LEVEL
-        value: "'DEBUG'"
+    - setting: LOG_AGGREGATOR_LEVEL
+      value: "'DEBUG'"
 ```
 
 Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes.
+
+## Add extra settings with `extra_settings_files`
+
+You can pass extra settings by specifying the additional settings files in the ConfigMaps or Secrets as the `extra_settings_files` parameter.
+
+The settings files passed via `extra_settings_files` will be mounted as the files under the `/etc/tower/conf.d`.
+
+| Name                 | Description          | Default   |
+| -------------------- | -------------------- | --------- |
+| extra_settings_files | Extra settings files | `{}`      |
+
+!!! note
+    If the same setting is set in multiple files in `extra_settings_files`, it would be difficult to predict which would be adopted since these files are loaded in arbitrary order that [`glob`](https://docs.python.org/3/library/glob.html) returns. For a reliable setting, do not include the same key in more than one file.
+
+Create ConfigMaps or Secrets that contain custom settings files (`*.py`).
+
+```python title="custom_job_settings.py"
+AWX_TASK_ENV = {
+    "HTTPS_PROXY": "http://proxy.example.com:3128",
+    "HTTP_PROXY": "http://proxy.example.com:3128",
+    "NO_PROXY": "127.0.0.1,localhost,.example.com"
+}
+GALAXY_TASK_ENV = {
+    "ANSIBLE_FORCE_COLOR": "false",
+    "GIT_SSH_COMMAND": "ssh -o StrictHostKeyChecking=no",
+}
+```
+
+```python title="custom_system_settings.py"
+REMOTE_HOST_HEADERS = [
+    "HTTP_X_FORWARDED_FOR",
+    "REMOTE_ADDR",
+    "REMOTE_HOST",
+]
+```
+
+```python title="custom_passwords.py"
+SUBSCRIPTIONS_PASSWORD = "my-super-secure-subscription-password123!"
+REDHAT_PASSWORD = "my-super-secure-redhat-password123!"
+```
+
+```bash title="Create ConfigMap and Secret"
+# Create ConfigMap
+kubectl create configmap my-custom-settings \
+  --from-file /PATH/TO/YOUR/custom_job_settings.py \
+  --from-file /PATH/TO/YOUR/custom_system_settings.py
+
+# Create Secret
+kubectl create secret generic my-custom-passwords \
+  --from-file /PATH/TO/YOUR/custom_passwords.py
+```
+
+Then specify them in the AWX CR spec. Here is an example configuration of `extra_settings_files` parameter.
+
+```yaml
+spec:
+  extra_settings_files:
+    configmaps:
+      - name: my-custom-settings        # The name of the ConfigMap
+        key: custom_job_settings.py     # The key in the ConfigMap, which means the file name
+      - name: my-custom-settings
+        key: custom_system_settings.py
+    secrets:
+      - name: my-custom-passwords       # The name of the Secret
+        key: custom_passwords.py        # The key in the Secret, which means the file name
+```
+
+!!! Warning "Restriction"
+    There are some restrictions on the ConfigMaps or Secrets used in `extra_settings_files`.
+
+    - The keys in ConfigMaps or Secrets MUST be the name of python files and MUST end with `.py`
+    - The keys in ConfigMaps or Secrets MUST consists of alphanumeric characters, `-`, `_` or `.`
+    - The keys in ConfigMaps or Secrets are converted to the following strings, which MUST not exceed 63 characters
+        - Keys in ConfigMaps: `<instance name>-<KEY>-configmap`
+        - Keys in Secrets: `<instance name>-<KEY>-secret`
+    - Following keys are reserved and MUST NOT be used in ConfigMaps or Secrets
+        - `credentials.py`
+        - `execution_environments.py`
+        - `ldap.py`
+
+    Refer to the Kubernetes documentations ([[1]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/config-map-v1/), [[2]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/), [[3]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/), [[4]](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)) for more information about character types and length restrictions.

docs/user-guide/advanced-configuration/horizontal-pod-autoscaler.md

@@ -0,0 +1,27 @@
+### Horizontal Pod Autoscaler (HPA)
+
+Horizontal Pod Autoscaler allows Kubernetes to scale the number of replicas of
+deployments in response to configured metrics.
+
+This feature conflicts with the operators ability to manage the number of static
+replicas to create for each deployment.
+
+The use of the settings below will tell the operator to not manage the replicas
+field on the identified deployments even if a replicas count has been set for those
+properties in the operator resource.
+
+| Name                   | Description                               | Default |
+| -----------------------| ----------------------------------------- | ------- |
+| web_manage_replicas    | Indicates operator should control the     | true    |
+|                        | replicas count for the web deployment.    |         |
+|                        |                                           |         |
+| task_manage_replicas   | Indicates operator should control the     | true    |
+|                        | replicas count for the task deployment.   |         |
+
+#### Recommended Settings for HPA
+
+Please see the Kubernetes documentation on how to configure the horizontal pod
+autoscaler.
+
+The values for optimal HPA are cluster and need specific so general guidelines
+are not available at this time.

docs/user-guide/advanced-configuration/mesh-ingress.md

@@ -86,7 +86,7 @@ spec:
 !!! tip
     AWX Operator supports both API groups `traefik.io` and `traefik.containo.us` in `ingress_api_version` for Traefik, but it is recommended to use `traefik.io` since `traefik.containo.us` is deprecated in Traefik v2.10 and is removed in Traefik v3. Refer to [Traefik documentation](https://doc.traefik.io/traefik/migration/v2/#v210) for more information about deprecation.
 
-    If you can't see any IngressRouteTCP resources by `kubectl` command after deploying mesh ingress, you should fully qualify the resource name with API group, `kubectl get ingressroutetcp.traefik.io` or `kubectl get traefik.containo.us` for example.
+    If you can't see any IngressRouteTCP resources by `kubectl` command after deploying mesh ingress, you should fully qualify the resource name with API group, `kubectl get ingressroutetcp.traefik.io` or `kubectl get ingressroutetcp.traefik.containo.us` for example.
 
 ### On Kubernetes with User managed Ingress
 

docs/user-guide/advanced-configuration/scaling-the-web-and-task-pods-independently.md

@@ -1,8 +1,13 @@
-#### Scaling the Web and Task Pods independently 
+#### Scaling the Web and Task Pods independently
 
 You can scale replicas up or down for each deployment by using the `web_replicas` or `task_replicas` respectively. You can scale all pods across both deployments by using `replicas` as well. The logic behind these CRD keys acts as such:
 
-- If you specify the `replicas` field, the key passed will scale both the `web` and `task` replicas to the same number. 
+- If you specify the `replicas` field, the key passed will scale both the `web` and `task` replicas to the same number.
 - If `web_replicas` or `task_replicas` is ever passed, it will override the existing `replicas` field on the specific deployment with the new key value.
 
-These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found in the [Assigning AWX pods to specific nodes](./assigning-awx-pods-to-specific-nodes.md) page. 
+These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found in the [Assigning AWX pods to specific nodes](./assigning-awx-pods-to-specific-nodes.md) page.
+
+##### Horizontal Pod Autoscaling
+
+The operator is capable of working with Kubernete's HPA capabilities.  See [Horizontal Pod Autoscaler](./horizontal-pod-autoscaler.md)
+documentation for more information.

docs/user-guide/database-configuration.md

@@ -27,6 +27,7 @@ stringData:
   username: <username to connect as>
   password: <password to connect with>
   sslmode: prefer
+  target_session_attrs: read-write
   type: unmanaged
 type: Opaque
 ```
@@ -37,6 +38,8 @@ type: Opaque
 
 **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`.
 
+**Note**: The variable `target_session_attrs` is only useful for `clustered external` databases. The allowed values are: `any` (default), `read-write`, `read-only`, `primary`, `standby` and `prefer-standby`, whereby only `read-write` and `primary` really make sense in AWX use, as you want to connect to a database node that offers write support.
+
 Once the secret is created, you can specify it on your spec:
 
 ```yaml

molecule/default/molecule.yml

@@ -21,6 +21,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         default_awx_version: "{{ lookup('url', 'https://api.github.com/repos/ansible/awx/releases/latest') | from_json | json_query('tag_name') }}"
@@ -30,6 +31,8 @@ provisioner:
         operator_image: ${OPERATOR_IMAGE:-""}
         operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"}
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"}
 verifier:

molecule/default/tasks/apply_awx_spec.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create or update the awx.ansible.com/v1alpha1.AWX
+- name: Create or update the awx.ansible.com/v1beta1.AWX
   k8s:
     state: present
     namespace: '{{ namespace }}'

molecule/default/tasks/awx_test.yml

@@ -1,98 +1,8 @@
 ---
 - include_tasks: apply_awx_spec.yml
 
-- name: Obtain generated admin password
-  k8s_info:
-    namespace: '{{ namespace }}'
-    kind: Secret
-    name: example-awx-admin-password
-  register: admin_pw_secret
-
-- block:
-    - name: Get web pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-web
-      register: awx_web_pod
-      when: not awx_version
-
-    - name: Get task pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-task
-      register: awx_task_pod
-      when: not awx_version
-
-    - name: Extract tags from images from web pod
-      set_fact:
-        web_image_tags: |
-          {{ awx_web_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - name: Extract tags from images from task pod
-      set_fact:
-        task_image_tags: |
-          {{ awx_task_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - fail:
-        msg: |
-          It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
-          This is an environment variable that is set via build arg when releasing awx-operator.
-      when:
-        - not awx_version
-        - default_awx_version not in web_image_tags
-        - default_awx_version not in task_image_tags
-
-    - name: Launch Demo Job Template
-      awx.awx.job_launch:
-        name: Demo Job Template
-        wait: yes
-        validate_certs: no
-        controller_host: localhost/awx/
-        controller_username: admin
-        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-  rescue:
-    - name: Get list of project updates and jobs
-      uri:
-        url: "http://localhost/awx/api/v2/{{ resource }}/"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      register: job_lists
-      loop:
-        - project_updates
-        - jobs
-      loop_control:
-        loop_var: resource
-
-    - name: Get all job and project details
-      uri:
-        url: "http://localhost{{ endpoint }}"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      loop: |
-        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
-      loop_control:
-        loop_var: endpoint
-
-    - name: Re-emit failure
-      vars:
-        failed_task:
-          result: '{{ ansible_failed_result }}'
-      fail:
-        msg: '{{ failed_task }}'
-
-- block:
+- name: Validate AWX deployment
+  block:
     - name: Look up details for this AWX instance
       k8s_info:
         namespace: "{{ namespace }}"
@@ -117,65 +27,90 @@
           - app.kubernetes.io/name = example-awx-task
       register: awx_task_pod
 
-    - name: Extract additional_labels from AWX spec
-      set_fact:
-        awx_additional_labels: >-
-          {{ this_awx.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
+    - name: Validate DEFAULT_AWX_VERSION
+      block:
+      - name: Extract tags from images from web pod
+        set_fact:
+          web_image_tags: |
+            {{ awx_web_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - name: Extract tags from images from task pod
+        set_fact:
+          task_image_tags: |
+            {{ awx_task_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - fail:
+          msg: |
+            It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
+            This is an environment variable that is set via build arg when releasing awx-operator.
+        when:
+          - default_awx_version not in web_image_tags
+          - default_awx_version not in task_image_tags
+      when: not awx_version
 
-    - name: Extract additional_labels from AWX web Pod
-      set_fact:
-        awx_web_pod_additional_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
-
-    - name: Extract additional_labels from AWX task Pod
-      set_fact:
-        awx_task_pod_additional_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
+    - name: Validate additional_labels
+      block:
+      - name: Extract additional_labels from AWX spec
+        set_fact:
+          awx_additional_labels: >-
+            {{ this_awx.resources[0].metadata.labels
               | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
               | list
-          }}
+            }}
 
-    - name: Assert AWX web Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX web Pod
+        set_fact:
+          awx_web_pod_additional_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+              | list
+            }}
 
-    - name: Assert AWX task Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX task Pod
+        set_fact:
+          awx_task_pod_additional_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+                | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+                | list
+            }}
 
-    - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_web_pod_extra_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Assert AWX web Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_additional_labels == awx_additional_labels
 
-    - name: AWX web Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_extra_labels == []
+      - name: Assert AWX task Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_additional_labels == awx_additional_labels
 
-    - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_task_pod_extra_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_web_pod_extra_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
 
-    - name: AWX task Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_extra_labels == []
+      - name: AWX web Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_extra_labels == []
+
+      - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_task_pod_extra_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
+
+      - name: AWX task Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_extra_labels == []
 
   rescue:
     - name: Re-emit failure
@@ -184,3 +119,124 @@
           result: '{{ ansible_failed_result }}'
       fail:
         msg: '{{ failed_task }}'
+
+- name: Obtain generated admin password
+  k8s_info:
+    namespace: '{{ namespace }}'
+    kind: Secret
+    name: example-awx-admin-password
+  register: admin_pw_secret
+
+- name: Wait for instance to be ready
+  uri:
+    url: "http://localhost/awx/api/v2/instances/?node_type=control&node_state=ready"
+    user: admin
+    password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+    force_basic_auth: yes
+  register: instances
+  until: instances['json']['count'] | int > 0
+  retries: 20
+  delay: 2
+
+- name: Validate demo job launch
+  block:
+    - name: Launch Demo Job Template
+      awx.awx.job_launch:
+        name: Demo Job Template
+        wait: yes
+        validate_certs: no
+        controller_host: localhost/awx/
+        controller_username: admin
+        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+
+  rescue:
+    - name: Create debug output directory
+      ansible.builtin.file:
+        path: '{{ debug_output_dir }}'
+        state: directory
+
+    - name: Get list of project updates and jobs
+      uri:
+        url: "http://localhost/awx/api/v2/{{ resource }}/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: job_lists
+      loop:
+        - project_updates
+        - jobs
+      loop_control:
+        loop_var: resource
+
+    - name: Store job_lists debug output
+      copy:
+        content: "{{ job_lists | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_lists.json"
+      when: store_debug_output | default(false)
+
+    - name: Get all job and project_update details
+      uri:
+        url: "http://localhost{{ endpoint }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
+      loop_control:
+        loop_var: endpoint
+      register: job_details
+
+    - name: Store job_details debug output
+      copy:
+        content: "{{ job_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_details.json"
+      when: store_debug_output | default(false)
+
+    - name: Get list of instances
+      uri:
+        url: "http://localhost/awx/api/v2/instances/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: instances_list
+
+    - name: Store instances_list debug output
+      copy:
+        content: "{{ instances_list | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_list.json"
+      when: store_debug_output | default(false)
+
+    - name: Get instances detail
+      uri:
+        url: "http://localhost{{ item }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ instances_list.json.results | map(attribute='url') }}
+      loop_control:
+        loop_var: item
+      register: instances_details
+
+    - name: Store instances_details debug output
+      copy:
+        content: "{{ instances_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_details.json"
+      when: store_debug_output | default(false)
+
+    ## TODO: figure out why this doesn't work
+    # - name: Store debug outputs
+    #   copy:
+    #     content: '{{ item }}'
+    #     dest: "{{ debug_output_dir }}/{{ item }}.json"
+    #   loop:
+    #     - job_lists
+    #     - job_details
+    #   when: store_debug_output | default(false)
+
+    - name: Re-emit failure
+      vars:
+        failed_task:
+          result: '{{ ansible_failed_result }}'
+      fail:
+        msg: '{{ failed_task }}'

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -13,6 +13,12 @@ spec:
 {% endif %}
 {% if awx_version %}
   image_version: {{ awx_version }}
+{% endif %}
+{% if awx_ee_image %}
+  control_plane_ee_image: {{ awx_ee_image }}
+  ee_images:
+  - image: {{ awx_ee_image }}
+    name: AWX EE
 {% endif %}
   ingress_type: ingress
   ingress_path: /awx
@@ -36,6 +42,10 @@ spec:
   additional_labels:
   - my/team
   - my/service
+  extra_settings:
+  - setting: LOG_AGGREGATOR_LEVEL
+    value: "'DEBUG'"
+  task_readiness_period: 15
 {% if additional_fields is defined %}
   {{ additional_fields | to_nice_yaml | indent(2) }}
 {% endif %}

molecule/default/utils/output_all_container_logs_for_pod.yml

@@ -0,0 +1,15 @@
+---
+- name: Get all container log in pod
+  kubernetes.core.k8s_log:
+    namespace: '{{ namespace }}'
+    name: '{{ item.metadata.name }}'
+    all_containers: true
+  register: all_container_logs
+
+- name: Store logs in file
+  ansible.builtin.copy:
+    content: "{{ all_container_logs.log_lines | join('\n') }}"
+    dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log'
+
+# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs
+# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported

molecule/default/utils/output_k8s_resources.yml

@@ -0,0 +1,29 @@
+---
+- name: Retrieve relevant k8s resources
+  kubernetes.core.k8s_info:
+    api_version: '{{ item.api_version }}'
+    kind: '{{ item.kind }}'
+    namespace: '{{ namespace }}'
+  loop:
+    - api_version: v1
+      kind: Pod
+    - api_version: apps/v1
+      kind: Deployment
+    - api_version: v1
+      kind: Secret
+    - api_version: v1
+      kind: ConfigMap
+    - api_version: "awx.ansible.com/v1beta1"
+      kind: AWX
+  register: debug_resources
+
+- name: debug print item.kind and item.metadata.name
+  debug:
+    msg: '{{ item.kind }}-{{ item.metadata.name }}'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"
+
+- name: Output gathered resource to files
+  ansible.builtin.copy:
+    content: '{{ item | to_nice_json }}'
+    dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"

molecule/default/verify.yml

@@ -10,53 +10,41 @@
     ctrl_label: control-plane=controller-manager
 
   tasks:
-    - block:
+    - name: Perform awx tests
+      block:
         - name: Import all test files from tasks/
-          include_tasks: '{{ item }}'
+          ansible.builtin.include_tasks: '{{ item }}'
           with_fileglob:
             - tasks/awx_test.yml
             - tasks/awx_replicas_test.yml
           tags:
             - always
       rescue:
-        - name: Retrieve relevant resources
-          k8s_info:
-            api_version: '{{ item.api_version }}'
-            kind: '{{ item.kind }}'
+        - name: Create debug output directory
+          ansible.builtin.file:
+            path: '{{ debug_output_dir }}'
+            state: directory
+          tags:
+            - always
+
+        - name: Gather and output K8s resources
+          ansible.builtin.include_tasks: utils/output_k8s_resources.yml
+          tags:
+            - always
+
+        - name: Get all pods
+          kubernetes.core.k8s_info:
+            api_version: v1
+            kind: Pod
             namespace: '{{ namespace }}'
-          loop:
-            - api_version: v1
-              kind: Pod
-            - api_version: apps/v1
-              kind: Deployment
-            - api_version: v1
-              kind: Secret
-            - api_version: v1
-              kind: ConfigMap
-          register: debug_resources
+          register: all_pods
           tags:
             - always
 
-        - name: Retrieve Pod logs
-          k8s_log:
-            name: '{{ item.metadata.name }}'
-            namespace: '{{ namespace }}'
-            container: awx-manager
-          loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}"
-          register: debug_logs
-          tags:
-            - always
-
-        - name: Output gathered resources
-          debug:
-            var: debug_resources
-          tags:
-            - always
-
-        - name: Output gathered logs
-          debug:
-            var: item.log_lines
-          loop: '{{ debug_logs.results }}'
+        - name: Get all container logs for all pods
+          ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml
+          loop: '{{ all_pods.resources }}'
+          ignore_errors: yes
           tags:
             - always
 
@@ -64,7 +52,7 @@
           vars:
             failed_task:
               result: '{{ ansible_failed_result }}'
-          fail:
+          ansible.builtin.fail:
             msg: '{{ failed_task }}'
           tags:
             - always

molecule/kind/molecule.yml

@@ -23,6 +23,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         ansible_python_interpreter: '{{ ansible_playbook_python }}'
@@ -34,6 +35,8 @@ provisioner:
         operator_pull_policy: "Never"
         kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig
     KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig

molecule/requirements.yml

@@ -2,8 +2,6 @@
 collections:
   - name: community.general
   - name: kubernetes.core
-    version: 2.3.2
   - name: operator_sdk.util
   - name: community.docker
-    version: 3.4.5
   - name: awx.awx

roles/backup/tasks/awx-cro.yml

@@ -31,5 +31,6 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir  }}/awx_object"
     content: "{{ awx_spec | to_yaml }}"

roles/backup/tasks/delete_backup.yml

@@ -3,5 +3,6 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       bash -c 'rm -rf {{ backup_dir  }}'

roles/backup/tasks/init.yml

@@ -77,10 +77,10 @@
   set_fact:
     _postgres_image: "{{ _custom_postgres_image | default(lookup('env', 'RELATED_IMAGE_AWX_POSTGRES')) | default(_default_postgres_image, true) }}"
 
-- name: Create management pod from templated deployment config
+- name: Create management pod from the template
   k8s:
     name: "{{ ansible_operator_meta.name }}-db-management"
-    kind: Deployment
+    kind: Pod
     state: present
     definition: "{{ lookup('template', 'management-pod.yml.j2') }}"
     wait: true

roles/backup/tasks/postgres.yml

@@ -60,6 +60,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       mkdir -p {{ backup_dir }}
 
@@ -67,6 +68,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       touch {{ backup_dir }}/tower.db
 
@@ -126,6 +128,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: |
       bash -c "
       function end_keepalive {

roles/backup/tasks/secrets.yml

@@ -20,9 +20,7 @@
 
 - name: Dump ingress tls secret names from awx spec and data into file
   include_tasks: dump_ingress_tls_secrets.yml
-  with_items:
-    - "{{ awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list }}"
-  when: awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list | length
+  with_items: "{{ awx_spec.spec['ingress_hosts'] | default([]) | selectattr('tls_secret', 'defined') | map(attribute='tls_secret') | list }}"
 
 - name: Dump receptor secret names and data into file
   include_tasks: dump_receptor_secrets.yml
@@ -52,6 +50,7 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir }}/secrets.yml"
     content: "{{ secrets | to_yaml }}"
   no_log: "{{ no_log }}"

roles/installer/defaults/main.yml

@@ -13,6 +13,7 @@ ingress_class_name: ''
 ingress_path: '/'
 ingress_path_type: 'Prefix'
 ingress_api_version: 'networking.k8s.io/v1'
+api_urlpattern_prefix: ''
 # Add annotations to the service account. Specify as literal block. E.g.:
 # service_account_annotations: |
 #   eks.amazonaws.com/role-arn: arn:aws:iam::<ACCOUNT_ID>:role/<IAM_ROLE_NAME>
@@ -491,13 +492,4 @@ nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
 nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
 
-# metrics-utility (github.com/ansible/metrics-utility)
-_metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
-_metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
-_metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
-_metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
-_metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
-_metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
-_metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
-_metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"
-_metrics_utility_cronjob_report_schedule: "{{ metrics_utility_cronjob_report_schedule | default('@monthly') }}"
+extra_settings_files: {}

roles/installer/tasks/database_configuration.yml

@@ -100,6 +100,7 @@
     awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
     awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
     awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] |  default('prefer'|b64encode) | b64decode }}"
+    awx_postgres_target_session_attrs: "{{ pg_config['resources'][0]['data']['target_session_attrs'] | default('') | b64decode }}"
   no_log: "{{ no_log }}"
 
 - name: Set database as managed

roles/installer/tasks/enable_metrics_utility.yml

@@ -1,23 +1,42 @@
 ---
-# Check to make sure provided pvc exists, error loudly if not.  Otherwise, the management pod will just stay in pending state forever.
-- name: Check provided PVC claim exists
-  kubernetes.core.k8s_info:
-    name: "{{ _metrics_utility_pvc_claim }}"
-    kind: PersistentVolumeClaim
-    namespace: "{{ ansible_operator_meta.namespace }}"
-  when:
-    - _metrics_utility_pvc_claim | length
+- name: Setup PVC if using directory ship target
+  block:
 
-- name: Create PVC for metrics-utility
-  kubernetes.core.k8s:
-    kind: PersistentVolumeClaim
-    definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+    # Check to make sure provided pvc exists
+    - name: Check provided PVC claim exists
+      kubernetes.core.k8s_info:
+        name: "{{ _metrics_utility_pvc_claim }}"
+        kind: PersistentVolumeClaim
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      when:
+        - _metrics_utility_pvc_claim | length
 
-- name: Create Kubernetes CronJobs for metrics-utility
+    - name: Create PVC for metrics-utility
+      kubernetes.core.k8s:
+        kind: PersistentVolumeClaim
+        definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+
+  when: _metrics_utility_ship_target == "directory"
+
+- name: Create default metrics-utility Kubernetes CronJobs
   kubernetes.core.k8s:
-    definition: "{{ lookup('template', item) }}"
+    definition: "{{ lookup('template', item.template) }}"
     apply: true
     wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
   loop:
-    - cronjobs/metrics-utility-gather.yaml.j2
-    - cronjobs/metrics-utility-report.yaml.j2
+    - {name: 'metrics-utility-gather', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+    - {name: 'metrics-utility-report', template: 'cronjobs/metrics-utility-report.yaml.j2'}
+
+- name: Create metrics-utility Kubernetes CronJob for Red Hat Hybrid Cloud Console
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', item.template) }}"
+    apply: true
+    wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
+    metrics_utility_ship_target: crc  # TODO - Update to console when changed
+  loop:
+    - {name: 'metrics-utility-gather-console', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+  when: _metrics_utility_console_enabled

roles/installer/tasks/migrate_schema.yml

@@ -24,7 +24,7 @@
 
     - name: Sanitize instance version
       set_fact:
-        version: "{{ version_check.stdout.split('+')[0] | trim }}"
+        version: "{{ version_check.stdout | replace('+', '-') | trim }}"
 
     # It is possible to do a wait on this task to create the job and wait
     # until it completes. Unfortunately, if the job doesn't wait finish within

roles/installer/tasks/resources_configuration.yml

@@ -267,8 +267,8 @@
           - status.phase=Running
       register: _new_pod
       until:
-        - "_new_pod['resources'] | length"
-        - "_new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | length"
+        - _new_pod['resources'] | length
+        - _new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | list | length
       retries: 60
       delay: 5
 

roles/installer/templates/common/volume_mounts/extra_settings_files.yaml.j2

@@ -0,0 +1,16 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  mountPath: "/etc/tower/conf.d/{{ configmap.key }}"
+  subPath: {{ configmap.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  mountPath: "/etc/tower/conf.d/{{ secret.key }}"
+  subPath: {{ secret.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}

roles/installer/templates/common/volumes/extra_settings_files.yaml.j2

@@ -0,0 +1,20 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  configMap:
+    name: {{ configmap.name }}
+    items:
+      - key: {{ configmap.key }}
+        path: {{ configmap.key }}
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  secret:
+    secretName: {{ secret.name }}
+    items:
+      - key: {{ secret.key }}
+        path: {{ secret.key }}
+{% endfor %}
+{% endif %}

roles/installer/templates/configmaps/config.yaml.j2

@@ -54,6 +54,10 @@ data:
 
     INTERNAL_API_URL = 'http://127.0.0.1:8052'
 
+{% if api_urlpattern_prefix | length > 0 %}
+    OPTIONAL_API_URLPATTERN_PREFIX = '{{ api_urlpattern_prefix }}'
+{% endif %}
+
     # Container environments don't like chroots
     AWX_PROOT_ENABLED = False
 
@@ -184,11 +188,19 @@ data:
                 alias /var/lib/awx/public/static/;
             }
 
+            location {{ (ingress_path + '/locales').replace('//', '/') }} {
+                alias /var/lib/awx/public/static/awx/locales;
+            }
+
             location {{ (ingress_path + '/favicon.ico').replace('//', '/') }} {
                 alias /var/lib/awx/public/static/media/favicon.ico;
             }
 
+{% if api_urlpattern_prefix | length > 0 %}
+            location ~ ^({{ (ingress_path + '/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/' + api_urlpattern_prefix + '/v2/websocket/').replace('//', '/') }}) {
+{% else %}
             location ~ ^({{ (ingress_path + '/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/websocket/').replace('//', '/') }}) {
+{% endif %}
                 # Pass request to the upstream alias
                 proxy_pass http://daphne;
                 # Require http version 1.1 to allow for upgrade requests

roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2

@@ -2,10 +2,10 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+  name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
     {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
@@ -16,7 +16,7 @@ spec:
       template:
         metadata:
           labels:
-            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
             {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
             {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
         spec:
@@ -30,7 +30,7 @@ spec:
 {% endfor %}
 {% endif %}
           containers:
-          - name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+          - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
             image: "{{ _metrics_utility_image }}"
             imagePullPolicy: "{{ image_pull_policy }}"
             resources:
@@ -41,9 +41,16 @@ spec:
               - /bin/sh
               - -c
               - metrics-utility gather_automation_controller_billing_data --ship --until=10m
+            env:
+            - name: METRICS_UTILITY_SHIP_TARGET
+              value: "{{ _metrics_utility_ship_target }}"
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
+{% if metrics_utility_secret is defined %}
+            - secretRef:
+                name: {{ metrics_utility_secret }}
+{% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
               mountPath: /metrics-utility
@@ -52,10 +59,15 @@ spec:
               mountPath: "/etc/tower/conf.d/credentials.py"
               subPath: credentials.py
               readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
             - name: {{ ansible_operator_meta.name }}-settings
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
           volumes:
           - name: {{ ansible_operator_meta.name }}-metrics-utility
             persistentVolumeClaim:
@@ -67,10 +79,17 @@ spec:
               items:
                 - key: credentials.py
                   path: 'credentials.py'
+          - name: "{{ secret_key_secret_name }}"
+            secret:
+              secretName: '{{ secret_key_secret_name }}'
+              items:
+                - key: secret_key
+                  path: SECRET_KEY
           - name: {{ ansible_operator_meta.name }}-settings
             configMap:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'
               items:
                 - key: settings
                   path: settings.py
+          {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=10) | trim }}
           restartPolicy: OnFailure

roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2

@@ -2,10 +2,10 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: {{ ansible_operator_meta.name }}-metrics-utility-report
+  name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
     {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
@@ -16,7 +16,7 @@ spec:
       template:
         metadata:
           labels:
-            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}'
             {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
             {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
         spec:
@@ -30,7 +30,7 @@ spec:
 {% endfor %}
 {% endif %}
           containers:
-          - name: {{ ansible_operator_meta.name }}-metrics-utility-report
+          - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }}
             image: "{{ _metrics_utility_image }}"
             imagePullPolicy: "{{ image_pull_policy }}"
             resources:
@@ -44,6 +44,10 @@ spec:
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
+{% if metrics_utility_secret is defined %}
+            - secretRef:
+                name: {{ metrics_utility_secret }}
+{% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
               mountPath: /metrics-utility
@@ -52,10 +56,15 @@ spec:
               mountPath: "/etc/tower/conf.d/credentials.py"
               subPath: credentials.py
               readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
             - name: {{ ansible_operator_meta.name }}-settings
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
           volumes:
           - name: {{ ansible_operator_meta.name }}-metrics-utility
             persistentVolumeClaim:
@@ -67,10 +76,17 @@ spec:
               items:
                 - key: credentials.py
                   path: 'credentials.py'
+          - name: "{{ secret_key_secret_name }}"
+            secret:
+              secretName: '{{ secret_key_secret_name }}'
+              items:
+                - key: secret_key
+                  path: SECRET_KEY
           - name: {{ ansible_operator_meta.name }}-settings
             configMap:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'
               items:
                 - key: settings
                   path: settings.py
+          {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=10) | trim }}
           restartPolicy: OnFailure

roles/installer/templates/deployments/task.yaml.j2

@@ -8,9 +8,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if task_replicas != '' %}
+{% if task_replicas != '' and task_manage_replicas | bool %}
   replicas: {{ task_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and task_manage_replicas | bool %}
   replicas: {{ replicas }}
 {% endif %}
   selector:
@@ -74,6 +74,25 @@ spec:
       priorityClassName: '{{ control_plane_priority_class }}'
 {% endif %}
       initContainers:
+{% if bundle_ca_crt  %}
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
         - name: init-database
           image: '{{ _image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -95,6 +114,11 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
+{% if bundle_ca_crt  %}
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+{% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
@@ -122,10 +146,6 @@ spec:
                 outcert=/etc/receptor/tls/receptor.crt \
                 notafter=$(date --iso-8601=seconds --utc --date "10 years") \
                 verify=yes
-{% if bundle_ca_crt  %}
-              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
-              update-ca-trust
-{% endif %}
 {% if init_container_extra_commands %}
               {{ init_container_extra_commands | indent(width=14) }}
 {% endif %}
@@ -148,10 +168,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if init_container_extra_volume_mounts -%}
             {{ init_container_extra_volume_mounts | indent(width=12, first=True) }}
@@ -244,7 +260,7 @@ spec:
             exec:
               command:
                 - /usr/bin/awx-manage
-                - check
+                - check_instance_ready
             initialDelaySeconds: {{ task_readiness_initial_delay }}
             periodSeconds: {{ task_readiness_period }}
             failureThreshold: {{ task_readiness_failure_threshold }}
@@ -254,10 +270,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: "{{ ansible_operator_meta.name }}-application-credentials"
               mountPath: "/etc/tower/conf.d/execution_environments.py"
@@ -279,6 +291,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -356,10 +369,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: "{{ ansible_operator_meta.name }}-default-receptor-config"
               mountPath: "/etc/receptor/receptor-default.conf"
@@ -428,6 +437,7 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -435,10 +445,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
@@ -588,6 +594,7 @@ spec:
             items:
               - key: redis_conf
                 path: redis.conf
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-redis-socket
           emptyDir: {}
         - name: {{ ansible_operator_meta.name }}-redis-data

roles/installer/templates/deployments/web.yaml.j2

@@ -9,9 +9,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if web_replicas != '' %}
+{% if web_replicas != '' and web_manage_replicas | bool %}
   replicas: {{ web_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and web_manage_replicas | bool %}
   replicas: {{ replicas }}
 {% endif %}
   selector:
@@ -80,7 +80,26 @@ spec:
       priorityClassName: '{{ control_plane_priority_class }}'
 {% endif %}
       initContainers:
-{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %}
+{% if bundle_ca_crt  %}
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
+{% if init_container_extra_commands %}
         - name: init
           image: '{{ _init_container_image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -89,21 +108,11 @@ spec:
             - /bin/sh
             - -c
             - |
-{% if bundle_ca_crt  %}
-              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
-              update-ca-trust
-{% endif %}
-{% if init_container_extra_commands %}
               {{ init_container_extra_commands | indent(width=14) }}
-{% endif %}
           volumeMounts:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if init_container_extra_volume_mounts -%}
             {{ init_container_extra_volume_mounts | indent(width=12, first=True) }}
@@ -191,10 +200,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: {{ ansible_operator_meta.name }}-uwsgi-config
               mountPath: "/etc/tower/uwsgi.ini"
@@ -231,6 +236,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-nginx-conf
               mountPath: /etc/nginx/nginx.conf
               subPath: nginx.conf
@@ -307,6 +313,7 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -314,10 +321,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
@@ -438,6 +441,7 @@ spec:
             items:
               - key: redis_conf
                 path: redis.conf
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-uwsgi-config
           configMap:
             name: {{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap

roles/installer/templates/jobs/migration.yaml.j2

@@ -8,7 +8,31 @@ metadata:
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
   template:
+    metadata:
+      labels:
+        {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=8) | trim }}
+        {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=8) | trim }}
     spec:
+{% if bundle_ca_crt %}
+      initContainers:
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
       containers:
         - name: "migration-job"
           image: '{{ _image }}'
@@ -29,6 +53,11 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
+{% if bundle_ca_crt %}
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+{% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
@@ -42,6 +71,34 @@ spec:
 {% for secret in image_pull_secrets %}
         - name: {{ secret }}
 {% endfor %}
+{% endif %}
+{% if task_node_selector %}
+      nodeSelector:
+        {{ task_node_selector | indent(width=8) }}
+{% elif node_selector %}
+      nodeSelector:
+        {{ node_selector | indent(width=8) }}
+{% endif %}
+{% if task_topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ task_topology_spread_constraints | indent(width=8) }}
+{% elif topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ topology_spread_constraints | indent(width=8) }}
+{% endif %}
+{% if task_tolerations %}
+      tolerations:
+        {{ task_tolerations | indent(width=8) }}
+{% elif tolerations %}
+      tolerations:
+        {{ tolerations | indent(width=8) }}
+{% endif %}
+{% if task_affinity %}
+      affinity:
+        {{ task_affinity | to_nice_yaml | indent(width=8) }}
+{% elif affinity %}
+      affinity:
+        {{ affinity | to_nice_yaml | indent(width=8) }}
 {% endif %}
       volumes:
         - name: "{{ ansible_operator_meta.name }}-application-credentials"
@@ -66,6 +123,17 @@ spec:
             items:
               - key: settings
                 path: settings.py
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
+{% if bundle_ca_crt %}
+        - name: "ca-trust-extracted"
+          emptyDir: {}
+        - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+          secret:
+            secretName: "{{ bundle_cacert_secret }}"
+            items:
+              - key: bundle-ca.crt
+                path: 'bundle-ca.crt'
+{% endif %}
 {% if development_mode | bool %}
         - name: awx-devel
           hostPath:

roles/installer/templates/settings/credentials.py.j2

@@ -10,6 +10,9 @@ DATABASES = {
         'OPTIONS': { 'sslmode': '{{ awx_postgres_sslmode }}',
 {% if awx_postgres_sslmode in ['verify-ca', 'verify-full'] %}
                      'sslrootcert': '{{ ca_trust_bundle }}',
+{% endif %}
+{% if awx_postgres_target_session_attrs %}
+                     'target_session_attrs': '{{ awx_postgres_target_session_attrs }}',
 {% endif %}
         },
     }
@@ -25,6 +28,9 @@ LISTENER_DATABASES = {
             'keepalives_count': {{ postgres_keepalives_count }},
 {% else %}
             'keepalives': 0,
+{% endif %}
+{% if awx_postgres_target_session_attrs %}
+            'target_session_attrs': '{{ awx_postgres_target_session_attrs }}',
 {% endif %}
         },
     }

roles/installer/vars/main.yml

@@ -8,3 +8,15 @@ supported_pg_version: 15
 _previous_upgraded_pg_version: 0
 old_postgres_pod: []
 _postgres_data_path: '/var/lib/pgsql/data/userdata'
+# metrics-utility (github.com/ansible/metrics-utility)
+_metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
+_metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
+_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
+_metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
+_metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
+_metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
+_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}"
+_metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
+_metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
+_metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"
+_metrics_utility_cronjob_report_schedule: "{{ metrics_utility_cronjob_report_schedule | default('@monthly') }}"

roles/mesh_ingress/defaults/main.yml

@@ -14,3 +14,8 @@ _image_pull_policy: Always
 image_pull_secrets: []
 
 finalizer_run: false
+
+node_selector: ''
+topology_spread_constraints: ''
+tolerations: ''
+affinity: {}

roles/mesh_ingress/templates/deployment.yml.j2

@@ -71,6 +71,22 @@ spec:
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccount: {{ ansible_operator_meta.name }}
+{% if node_selector %}
+      nodeSelector:
+        {{ node_selector | indent(width=8) }}
+{% endif %}
+{% if topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ topology_spread_constraints | indent(width=8) }}
+{% endif %}
+{% if tolerations %}
+      tolerations:
+        {{ tolerations | indent(width=8) }}
+{% endif %}
+{% if affinity %}
+      affinity:
+        {{ affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
       volumes:
       - name: {{ ansible_operator_meta.name }}-receptor-tls
       - name: {{ ansible_operator_meta.name }}-receptor-ca

roles/restore/defaults/main.yml

@@ -39,4 +39,6 @@ additional_labels: []
 
 # Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
 set_self_labels: true
+
+spec_overrides: {}
 ...

roles/restore/tasks/deploy_awx.yml

@@ -1,5 +1,10 @@
 ---
 
+- name: Combine spec_overrides with spec
+  set_fact:
+    spec: "{{ spec | default({}) | combine(spec_overrides) }}"
+  no_log: "{{ no_log }}"
+
 - name: Deploy AWX
   k8s:
     state: "{{ state | default('present') }}"

roles/restore/tasks/import_vars.yml

@@ -12,6 +12,7 @@
     k8s_cp:
       namespace: "{{ backup_pvc_namespace }}"
       pod: "{{ ansible_operator_meta.name }}-db-management"
+      container: "{{ ansible_operator_meta.name }}-db-management"
       remote_path: "{{ backup_dir  }}/awx_object"
       local_path: "{{ tmp_spec.path }}"
       state: from_pod

roles/restore/tasks/init.yml

@@ -86,10 +86,10 @@
   set_fact:
     _postgres_image: "{{ _custom_postgres_image | default(lookup('env', 'RELATED_IMAGE_AWX_POSTGRES'))  | default(_default_postgres_image, true) }}"
 
-- name: Create management pod from templated deployment config
+- name: Create management pod from the template
   k8s:
     name: "{{ ansible_operator_meta.name }}-db-management"
-    kind: Deployment
+    kind: Pod
     state: present
     definition: "{{ lookup('template', 'management-pod.yml.j2') }}"
     wait: true
@@ -98,6 +98,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       bash -c "stat {{ backup_dir }}"
   register: stat_backup_dir

roles/restore/tasks/postgres.yml

@@ -115,6 +115,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: |
       bash -c "
       function end_keepalive {

roles/restore/tasks/secrets.yml

@@ -10,6 +10,7 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir  }}/secrets.yml"
     local_path: "{{ tmp_secrets.path }}"
     state: from_pod