Wait for instance ready in molecule test (#1901)

Sometimes a job is launched through the web api
before the instance is in a ready state. This throws
a 500 internal server error, causing CI to fail.

Adds a task to query the instances endpoint
and check that at least one control node is
in a ready state.

Signed-off-by: Seth Foster <fosterbseth@gmail.com>

.github/workflows/reusable-nox.yml

@@ -18,7 +18,7 @@ jobs:
         - name: Check out repo
           uses: actions/checkout@v4
         - name: Setup nox
-          uses: wntrblm/nox@2024.03.02
+          uses: wntrblm/nox@2024.04.15
           with:
               python-versions: "${{ matrix.python-versions }}"
         - name: "Run nox -s ${{ matrix.session }}"

.helm/starter/templates/postgres-config.yaml

@@ -12,6 +12,7 @@ stringData:
   username: {{ .username }}
   password: {{ .password }}
   sslmode: {{ .sslmode }}
+  target_session_attrs: {{ .target_session_attrs | default "any" }}
   type: {{ .type }}
 type: Opaque
 {{- end }}

config/crd/bases/awx.ansible.com_awxmeshingresses.yaml

@@ -75,6 +75,381 @@ spec:
               ingress_controller:
                 description: Special configuration for specific Ingress Controllers
                 type: string
+              node_selector:
+                description: Assign the Mesh Ingress Pod to the specified node.
+                type: string
+              tolerations:
+                description: Scheduling tolerations for the Mesh Ingress instance.
+                type: string
+              topology_spread_constraints:
+                description: Topology spread constraints for the Mesh Ingress instance.
+                type: string
+              affinity:
+                description: Scheduling constraints to apply to the Pod definition
+                properties:
+                  nodeAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            preference:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - preference
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        properties:
+                          nodeSelectorTerms:
+                            items:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            type: array
+                        required:
+                        - nodeSelectorTerms
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                  podAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaceSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaceSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                  podAntiAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaceSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaceSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                type: object
           status:
             description: Status defines the observed state of AWXMeshIngress
             type: object

config/crd/bases/awx.ansible.com_awxrestores.yaml

@@ -94,6 +94,11 @@ spec:
               postgres_image_version:
                 description: PostgreSQL container image version to use
                 type: string
+              spec_overrides:
+                description: Overrides for the AWX spec
+                # type: string
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
               image_pull_policy:
                 description: The image pull policy
                 type: string

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -144,6 +144,9 @@ spec:
               ingress_controller:
                 description: Special configuration for specific Ingress Controllers
                 type: string
+              api_urlpattern_prefix:
+                description: An optional configuration to add a prefix in the API URL path
+                type: string
               loadbalancer_protocol:
                 description: Protocol to use for the loadbalancer
                 type: string
@@ -1571,10 +1574,18 @@ spec:
                 description: Number of web instance replicas
                 type: integer
                 format: int32
+              web_manage_replicas:
+                description: Enables operator control of replicas count for the web deployment when set to 'true'
+                type: boolean
+                default: true
               task_replicas:
                 description: Number of task instance replicas
                 type: integer
                 format: int32
+              task_manage_replicas:
+                description: Enables operator control of replicas count for the task deployment when set to 'true'
+                type: boolean
+                default: true
               web_liveness_initial_delay:
                 description: Initial delay before starting liveness checks on web pod
                 type: integer
@@ -1901,6 +1912,28 @@ spec:
                       x-kubernetes-preserve-unknown-fields: true
                   type: object
                 type: array
+              extra_settings_files:
+                description: Extra ConfigMaps or Secrets of settings files to specify for AWX
+                properties:
+                  configmaps:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                  secrets:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                type: object
               no_log:
                 description: Configure no_log for no_log tasks
                 type: boolean

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -58,6 +58,31 @@ spec:
         path: ingress_controller
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Node Selector
+        path: node_selector
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Tolerations
+        path: tolerations
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Topology Spread Constraints
+        path: topology_spread_constraints
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Affinity
+        path: affinity
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Optional API URLPATTERN Prefix
+        path: api_urlpattern_prefix
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
       - displayName: Image Pull Secrets
         path: image_pull_secrets
         x-descriptors:
@@ -213,6 +238,10 @@ spec:
         path: postgres_image_version
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: AWX Spec Overrides
+        path: spec_overrides
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Image Pull Policy
         path: image_pull_policy
         x-descriptors:
@@ -957,6 +986,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Extra Settings Files
+        path: extra_settings_files
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: No Log Configuration
         path: no_log
         x-descriptors:

docs/requirements.txt

@@ -94,7 +94,7 @@ mkdocs==1.5.3
     #   mkdocs-minify-plugin
     #   mkdocs-monorepo-plugin
     #   mkdocstrings
-mkdocs-ansible==24.3.0
+mkdocs-ansible==24.3.1
     # via -r requirements.in
 mkdocs-autorefs==0.5.0
     # via mkdocstrings

docs/user-guide/advanced-configuration/assigning-awx-pods-to-specific-nodes.md

@@ -88,3 +88,8 @@ spec:
               - S2
           topologyKey: topology.kubernetes.io/zone
 ```
+
+#### Special Note on DB-Migration Job Scheduling
+
+For the **db-migration job**, which applies database migrations at cluster startup, you can specify scheduling settings using the `task_*` configurations such as `task_node_selector`, `task_tolerations`, etc.  
+If these task-specific settings are not defined, the job will automatically use the global AWX configurations like `node_selector` and `tolerations`.

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -1,4 +1,4 @@
-#### Custom Volume and Volume Mount Options
+# Custom Volume and Volume Mount Options
 
 In a scenario where custom volumes and volume mounts are required to either overwrite defaults or mount configuration files.
 
@@ -12,7 +12,6 @@ In a scenario where custom volumes and volume mounts are required to either over
 | init_container_extra_volume_mounts | Specify volume mounts to be added to Init container      | ''      |
 | init_container_extra_commands      | Specify additional commands for Init container           | ''      |
 
-
 !!! warning
     The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
 
@@ -31,10 +30,8 @@ data:
     remote_tmp = /tmp
     [ssh_connection]
     ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
-  custom.py:  |
-    INSIGHTS_URL_BASE = "example.org"
-    AWX_CLEANUP_PATHS = True
 ```
+
 Example spec file for volumes and volume mounts
 
 ```yaml
@@ -49,13 +46,6 @@ spec:
           - key: ansible.cfg
             path: ansible.cfg
         name: <resourcename>-extra-config
-    - name: custom-py
-      configMap:
-        defaultMode: 420
-        items:
-          - key: custom.py
-            path: custom.py
-        name: <resourcename>-extra-config
     - name: shared-volume
       persistentVolumeClaim:
         claimName: my-external-volume-claim
@@ -73,24 +63,13 @@ spec:
     - name: ansible-cfg
       mountPath: /etc/ansible/ansible.cfg
       subPath: ansible.cfg
-
-  web_extra_volume_mounts: |
-    - name: custom-py
-      mountPath: /etc/tower/conf.d/custom.py
-      subPath: custom.py
-
-  task_extra_volume_mounts: |
-    - name: custom-py
-      mountPath: /etc/tower/conf.d/custom.py
-      subPath: custom.py
-    - name: shared-volume
-      mountPath: /shared
 ```
 
 !!! warning
     **Volume and VolumeMount names cannot contain underscores(_)**
 
-##### Custom UWSGI Configuration
+## Custom UWSGI Configuration
+
 We allow the customization of two UWSGI parameters:
 
 * [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5)
@@ -110,7 +89,7 @@ requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related nginx
 configuration.
 
-##### Custom Nginx Configuration
+## Custom Nginx Configuration
 
 Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf.
 
@@ -131,20 +110,62 @@ may allow the web pods to handle more "bursty" request patterns if many
 requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related uwsgi
 configuration.
+
 * [worker_processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes) with `nginx_worker_processes` (default of 1)
 * [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) with `nginx_worker_cpu_affinity` (default "auto")
 * [worker_connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) with `nginx_worker_connections` (minimum of 1024)
 * [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size)
 
+## Custom Logos
 
-##### Custom Favicon
+You can use custom volume mounts to mount in your own logos to be displayed instead of the AWX logo.
+There are two different logos, one to be displayed on page headers, and one for the login screen.
 
-You can use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
+First, create configmaps for the logos from local `logo-login.svg` and `logo-header.svg` files.
-
-First, Create the configmap from a local favicon.ico file.
 
 ```bash
-$ oc create configmap favicon-configmap --from-file favicon.ico
+kubectl create configmap logo-login-configmap --from-file logo-login.svg
+kubectl create configmap logo-header-configmap --from-file logo-header.svg
+```
+
+Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
+
+```yaml
+---
+spec:
+  ...
+  extra_volumes: |
+    - name: logo-login
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-login.svg
+            path: logo-login.svg
+        name: logo-login-configmap
+    - name: logo-header
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-header.svg
+            path: logo-header.svg
+        name: logo-header-configmap
+  web_extra_volume_mounts: |
+    - name: logo-login
+      mountPath: /var/lib/awx/public/static/media/logo-login.svg
+      subPath: logo-login.svg
+    - name: logo-header
+      mountPath: /var/lib/awx/public/static/media/logo-header.svg
+      subPath: logo-header.svg
+```
+
+## Custom Favicon
+
+You can also use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
+
+First, create the configmap from a local `favicon.ico` file.
+
+```bash
+kubectl create configmap favicon-configmap --from-file favicon.ico
 ```
 
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
@@ -166,3 +187,7 @@ spec:
       mountPath: /var/lib/awx/public/static/media/favicon.ico
       subPath: favicon.ico
 ```
+
+## Custom AWX Configuration
+
+Refer to the [Extra Settings](./extra-settings.md) documentation for customizing the AWX configuration.

docs/user-guide/advanced-configuration/extra-settings.md

@@ -1,30 +1,119 @@
-#### Extra Settings
+# Extra Settings
 
-With`extra_settings`, you can pass multiple custom settings via the `awx-operator`. The parameter `extra_settings`  will be appended to the `/etc/tower/settings.py` and can be an alternative to the `extra_volumes` parameter.
+With `extra_settings` and `extra_settings_files`, you can pass multiple custom settings to AWX via the AWX Operator.
 
-| Name           | Description    | Default |
+!!! note
-| -------------- | -------------- | ------- |
+    Parameters configured in `extra_settings` or `extra_settings_files` are set as read-only settings in AWX. As a result, they cannot be changed in the UI after deployment.
-| extra_settings | Extra settings | ''      |
 
-**Note:** Parameters configured in `extra_settings` are set as read-only settings in AWX.  As a result, they cannot be changed in the UI after deployment. If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec.  
+    If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec (for `extra_settings`) or corresponding ConfigMap or Secret (for `extra_settings_files`). After updating ConfigMap or Secret, you need to restart the AWX pods to apply the changes.
+
+!!! note
+    If the same setting is set in both `extra_settings` and `extra_settings_files`, the setting in `extra_settings_files` will take precedence.
+
+## Add extra settings with `extra_settings`
+
+You can pass extra settings by specifying the pair of the setting name and value as the `extra_settings` parameter.
+
+The settings passed via `extra_settings` will be appended to the `/etc/tower/settings.py`.
+
+| Name           | Description    | Default   |
+| -------------- | -------------- | --------- |
+| extra_settings | Extra settings | `[]`      |
 
 Example configuration of `extra_settings` parameter
 
 ```yaml
-  spec:
+spec:
-    extra_settings:
+  extra_settings:
-      - setting: MAX_PAGE_SIZE
+    - setting: MAX_PAGE_SIZE
-        value: "500"
+      value: "500"
 
-      - setting: AUTH_LDAP_BIND_DN
+    - setting: AUTH_LDAP_BIND_DN
-        value: "cn=admin,dc=example,dc=com"
+      value: "cn=admin,dc=example,dc=com"
 
-      - setting: LOG_AGGREGATOR_LEVEL
+    - setting: LOG_AGGREGATOR_LEVEL
-        value: "'DEBUG'"
+      value: "'DEBUG'"
 ```
 
 Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes.
 
-!!! tip
+## Add extra settings with `extra_settings_files`
-    Alternatively, you can pass any additional settings by mounting ConfigMaps or Secrets of the python files (`*.py`) that contain custom settings to under `/etc/tower/conf.d/` in the web and task pods.
+
-    See the example of `custom.py` in the [Custom Volume and Volume Mount Options](custom-volume-and-volume-mount-options.md) section.
+You can pass extra settings by specifying the additional settings files in the ConfigMaps or Secrets as the `extra_settings_files` parameter.
+
+The settings files passed via `extra_settings_files` will be mounted as the files under the `/etc/tower/conf.d`.
+
+| Name                 | Description          | Default   |
+| -------------------- | -------------------- | --------- |
+| extra_settings_files | Extra settings files | `{}`      |
+
+!!! note
+    If the same setting is set in multiple files in `extra_settings_files`, it would be difficult to predict which would be adopted since these files are loaded in arbitrary order that [`glob`](https://docs.python.org/3/library/glob.html) returns. For a reliable setting, do not include the same key in more than one file.
+
+Create ConfigMaps or Secrets that contain custom settings files (`*.py`).
+
+```python title="custom_job_settings.py"
+AWX_TASK_ENV = {
+    "HTTPS_PROXY": "http://proxy.example.com:3128",
+    "HTTP_PROXY": "http://proxy.example.com:3128",
+    "NO_PROXY": "127.0.0.1,localhost,.example.com"
+}
+GALAXY_TASK_ENV = {
+    "ANSIBLE_FORCE_COLOR": "false",
+    "GIT_SSH_COMMAND": "ssh -o StrictHostKeyChecking=no",
+}
+```
+
+```python title="custom_system_settings.py"
+REMOTE_HOST_HEADERS = [
+    "HTTP_X_FORWARDED_FOR",
+    "REMOTE_ADDR",
+    "REMOTE_HOST",
+]
+```
+
+```python title="custom_passwords.py"
+SUBSCRIPTIONS_PASSWORD = "my-super-secure-subscription-password123!"
+REDHAT_PASSWORD = "my-super-secure-redhat-password123!"
+```
+
+```bash title="Create ConfigMap and Secret"
+# Create ConfigMap
+kubectl create configmap my-custom-settings \
+  --from-file /PATH/TO/YOUR/custom_job_settings.py \
+  --from-file /PATH/TO/YOUR/custom_system_settings.py
+
+# Create Secret
+kubectl create secret generic my-custom-passwords \
+  --from-file /PATH/TO/YOUR/custom_passwords.py
+```
+
+Then specify them in the AWX CR spec. Here is an example configuration of `extra_settings_files` parameter.
+
+```yaml
+spec:
+  extra_settings_files:
+    configmaps:
+      - name: my-custom-settings        # The name of the ConfigMap
+        key: custom_job_settings.py     # The key in the ConfigMap, which means the file name
+      - name: my-custom-settings
+        key: custom_system_settings.py
+    secrets:
+      - name: my-custom-passwords       # The name of the Secret
+        key: custom_passwords.py        # The key in the Secret, which means the file name
+```
+
+!!! Warning "Restriction"
+    There are some restrictions on the ConfigMaps or Secrets used in `extra_settings_files`.
+
+    - The keys in ConfigMaps or Secrets MUST be the name of python files and MUST end with `.py`
+    - The keys in ConfigMaps or Secrets MUST consists of alphanumeric characters, `-`, `_` or `.`
+    - The keys in ConfigMaps or Secrets are converted to the following strings, which MUST not exceed 63 characters
+        - Keys in ConfigMaps: `<instance name>-<KEY>-configmap`
+        - Keys in Secrets: `<instance name>-<KEY>-secret`
+    - Following keys are reserved and MUST NOT be used in ConfigMaps or Secrets
+        - `credentials.py`
+        - `execution_environments.py`
+        - `ldap.py`
+
+    Refer to the Kubernetes documentations ([[1]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/config-map-v1/), [[2]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/), [[3]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/), [[4]](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)) for more information about character types and length restrictions.

docs/user-guide/advanced-configuration/horizontal-pod-autoscaler.md

@@ -0,0 +1,27 @@
+### Horizontal Pod Autoscaler (HPA)
+
+Horizontal Pod Autoscaler allows Kubernetes to scale the number of replicas of
+deployments in response to configured metrics.
+
+This feature conflicts with the operators ability to manage the number of static
+replicas to create for each deployment.
+
+The use of the settings below will tell the operator to not manage the replicas
+field on the identified deployments even if a replicas count has been set for those
+properties in the operator resource.
+
+| Name                   | Description                               | Default |
+| -----------------------| ----------------------------------------- | ------- |
+| web_manage_replicas    | Indicates operator should control the     | true    |
+|                        | replicas count for the web deployment.    |         |
+|                        |                                           |         |
+| task_manage_replicas   | Indicates operator should control the     | true    |
+|                        | replicas count for the task deployment.   |         |
+
+#### Recommended Settings for HPA
+
+Please see the Kubernetes documentation on how to configure the horizontal pod
+autoscaler.
+
+The values for optimal HPA are cluster and need specific so general guidelines
+are not available at this time.

docs/user-guide/advanced-configuration/scaling-the-web-and-task-pods-independently.md

@@ -6,3 +6,8 @@ You can scale replicas up or down for each deployment by using the `web_replicas
 - If `web_replicas` or `task_replicas` is ever passed, it will override the existing `replicas` field on the specific deployment with the new key value.
 
 These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found in the [Assigning AWX pods to specific nodes](./assigning-awx-pods-to-specific-nodes.md) page.
+
+##### Horizontal Pod Autoscaling
+
+The operator is capable of working with Kubernete's HPA capabilities.  See [Horizontal Pod Autoscaler](./horizontal-pod-autoscaler.md)
+documentation for more information.

docs/user-guide/database-configuration.md

@@ -27,6 +27,7 @@ stringData:
   username: <username to connect as>
   password: <password to connect with>
   sslmode: prefer
+  target_session_attrs: read-write
   type: unmanaged
 type: Opaque
 ```
@@ -37,6 +38,8 @@ type: Opaque
 
 **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`.
 
+**Note**: The variable `target_session_attrs` is only useful for `clustered external` databases. The allowed values are: `any` (default), `read-write`, `read-only`, `primary`, `standby` and `prefer-standby`, whereby only `read-write` and `primary` really make sense in AWX use, as you want to connect to a database node that offers write support.
+
 Once the secret is created, you can specify it on your spec:
 
 ```yaml

molecule/default/tasks/awx_test.yml

@@ -127,6 +127,17 @@
     name: example-awx-admin-password
   register: admin_pw_secret
 
+- name: Wait for instance to be ready
+  uri:
+    url: "http://localhost/awx/api/v2/instances/?node_type=control&node_state=ready"
+    user: admin
+    password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+    force_basic_auth: yes
+  register: instances
+  until: instances['json']['count'] | int > 0
+  retries: 20
+  delay: 2
+
 - name: Validate demo job launch
   block:
     - name: Launch Demo Job Template
@@ -139,6 +150,11 @@
         controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
 
   rescue:
+    - name: Create debug output directory
+      ansible.builtin.file:
+        path: '{{ debug_output_dir }}'
+        state: directory
+
     - name: Get list of project updates and jobs
       uri:
         url: "http://localhost/awx/api/v2/{{ resource }}/"
@@ -158,7 +174,7 @@
         dest: "{{ debug_output_dir }}/job_lists.json"
       when: store_debug_output | default(false)
 
-    - name: Get all job and project details
+    - name: Get all job and project_update details
       uri:
         url: "http://localhost{{ endpoint }}"
         user: admin
@@ -176,6 +192,38 @@
         dest: "{{ debug_output_dir }}/job_details.json"
       when: store_debug_output | default(false)
 
+    - name: Get list of instances
+      uri:
+        url: "http://localhost/awx/api/v2/instances/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: instances_list
+
+    - name: Store instances_list debug output
+      copy:
+        content: "{{ instances_list | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_list.json"
+      when: store_debug_output | default(false)
+
+    - name: Get instances detail
+      uri:
+        url: "http://localhost{{ item }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ instances_list.json.results | map(attribute='url') }}
+      loop_control:
+        loop_var: item
+      register: instances_details
+
+    - name: Store instances_details debug output
+      copy:
+        content: "{{ instances_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_details.json"
+      when: store_debug_output | default(false)
+
     ## TODO: figure out why this doesn't work
     # - name: Store debug outputs
     #   copy:

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -42,6 +42,10 @@ spec:
   additional_labels:
   - my/team
   - my/service
+  extra_settings:
+  - setting: LOG_AGGREGATOR_LEVEL
+    value: "'DEBUG'"
+  task_readiness_period: 15
 {% if additional_fields is defined %}
   {{ additional_fields | to_nice_yaml | indent(2) }}
 {% endif %}

molecule/requirements.yml

@@ -2,8 +2,6 @@
 collections:
   - name: community.general
   - name: kubernetes.core
-    version: 2.3.2
   - name: operator_sdk.util
   - name: community.docker
-    version: 3.4.5
   - name: awx.awx

roles/backup/tasks/awx-cro.yml

@@ -31,5 +31,6 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir  }}/awx_object"
     content: "{{ awx_spec | to_yaml }}"

roles/backup/tasks/delete_backup.yml

@@ -3,5 +3,6 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       bash -c 'rm -rf {{ backup_dir  }}'

roles/backup/tasks/init.yml

@@ -77,10 +77,10 @@
   set_fact:
     _postgres_image: "{{ _custom_postgres_image | default(lookup('env', 'RELATED_IMAGE_AWX_POSTGRES')) | default(_default_postgres_image, true) }}"
 
-- name: Create management pod from templated deployment config
+- name: Create management pod from the template
   k8s:
     name: "{{ ansible_operator_meta.name }}-db-management"
-    kind: Deployment
+    kind: Pod
     state: present
     definition: "{{ lookup('template', 'management-pod.yml.j2') }}"
     wait: true

roles/backup/tasks/postgres.yml

@@ -60,6 +60,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       mkdir -p {{ backup_dir }}
 
@@ -67,6 +68,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       touch {{ backup_dir }}/tower.db
 
@@ -126,6 +128,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: |
       bash -c "
       function end_keepalive {

roles/backup/tasks/secrets.yml

@@ -50,6 +50,7 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir }}/secrets.yml"
     content: "{{ secrets | to_yaml }}"
   no_log: "{{ no_log }}"

roles/installer/defaults/main.yml

@@ -13,6 +13,7 @@ ingress_class_name: ''
 ingress_path: '/'
 ingress_path_type: 'Prefix'
 ingress_api_version: 'networking.k8s.io/v1'
+api_urlpattern_prefix: ''
 # Add annotations to the service account. Specify as literal block. E.g.:
 # service_account_annotations: |
 #   eks.amazonaws.com/role-arn: arn:aws:iam::<ACCOUNT_ID>:role/<IAM_ROLE_NAME>
@@ -491,15 +492,4 @@ nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
 nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
 
-# metrics-utility (github.com/ansible/metrics-utility)
+extra_settings_files: {}
-_metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
-_metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
-_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
-_metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
-_metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
-_metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
-_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}"
-_metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
-_metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
-_metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"
-_metrics_utility_cronjob_report_schedule: "{{ metrics_utility_cronjob_report_schedule | default('@monthly') }}"

roles/installer/tasks/database_configuration.yml

@@ -100,6 +100,7 @@
     awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
     awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
     awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] |  default('prefer'|b64encode) | b64decode }}"
+    awx_postgres_target_session_attrs: "{{ pg_config['resources'][0]['data']['target_session_attrs'] | default('') | b64decode }}"
   no_log: "{{ no_log }}"
 
 - name: Set database as managed

roles/installer/tasks/migrate_schema.yml

@@ -24,7 +24,7 @@
 
     - name: Sanitize instance version
       set_fact:
-        version: "{{ version_check.stdout.split('+')[0] | trim }}"
+        version: "{{ version_check.stdout | replace('+', '-') | trim }}"
 
     # It is possible to do a wait on this task to create the job and wait
     # until it completes. Unfortunately, if the job doesn't wait finish within

roles/installer/templates/common/volume_mounts/extra_settings_files.yaml.j2

@@ -0,0 +1,16 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  mountPath: "/etc/tower/conf.d/{{ configmap.key }}"
+  subPath: {{ configmap.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  mountPath: "/etc/tower/conf.d/{{ secret.key }}"
+  subPath: {{ secret.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}

roles/installer/templates/common/volumes/extra_settings_files.yaml.j2

@@ -0,0 +1,20 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  configMap:
+    name: {{ configmap.name }}
+    items:
+      - key: {{ configmap.key }}
+        path: {{ configmap.key }}
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  secret:
+    secretName: {{ secret.name }}
+    items:
+      - key: {{ secret.key }}
+        path: {{ secret.key }}
+{% endfor %}
+{% endif %}

roles/installer/templates/configmaps/config.yaml.j2

@@ -54,6 +54,10 @@ data:
 
     INTERNAL_API_URL = 'http://127.0.0.1:8052'
 
+{% if api_urlpattern_prefix | length > 0 %}
+    OPTIONAL_API_URLPATTERN_PREFIX = '{{ api_urlpattern_prefix }}'
+{% endif %}
+
     # Container environments don't like chroots
     AWX_PROOT_ENABLED = False
 
@@ -184,11 +188,19 @@ data:
                 alias /var/lib/awx/public/static/;
             }
 
+            location {{ (ingress_path + '/locales').replace('//', '/') }} {
+                alias /var/lib/awx/public/static/awx/locales;
+            }
+
             location {{ (ingress_path + '/favicon.ico').replace('//', '/') }} {
                 alias /var/lib/awx/public/static/media/favicon.ico;
             }
 
+{% if api_urlpattern_prefix | length > 0 %}
+            location ~ ^({{ (ingress_path + '/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/' + api_urlpattern_prefix + '/v2/websocket/').replace('//', '/') }}) {
+{% else %}
             location ~ ^({{ (ingress_path + '/websocket/').replace('//', '/') }}|{{ (ingress_path + '/api/websocket/').replace('//', '/') }}) {
+{% endif %}
                 # Pass request to the upstream alias
                 proxy_pass http://daphne;
                 # Require http version 1.1 to allow for upgrade requests

roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2

@@ -47,9 +47,9 @@ spec:
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
-{% if _metrics_utility_secret is defined %}
+{% if metrics_utility_secret is defined %}
             - secretRef:
-                name: {{ _metrics_utility_secret }}
+                name: {{ metrics_utility_secret }}
 {% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
@@ -59,10 +59,15 @@ spec:
               mountPath: "/etc/tower/conf.d/credentials.py"
               subPath: credentials.py
               readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
             - name: {{ ansible_operator_meta.name }}-settings
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
           volumes:
           - name: {{ ansible_operator_meta.name }}-metrics-utility
             persistentVolumeClaim:
@@ -74,10 +79,17 @@ spec:
               items:
                 - key: credentials.py
                   path: 'credentials.py'
+          - name: "{{ secret_key_secret_name }}"
+            secret:
+              secretName: '{{ secret_key_secret_name }}'
+              items:
+                - key: secret_key
+                  path: SECRET_KEY
           - name: {{ ansible_operator_meta.name }}-settings
             configMap:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'
               items:
                 - key: settings
                   path: settings.py
+          {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=10) | trim }}
           restartPolicy: OnFailure

roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2

@@ -44,9 +44,9 @@ spec:
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
-{% if _metrics_utility_secret is defined %}
+{% if metrics_utility_secret is defined %}
             - secretRef:
-                name: {{ _metrics_utility_secret }}
+                name: {{ metrics_utility_secret }}
 {% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
@@ -56,10 +56,15 @@ spec:
               mountPath: "/etc/tower/conf.d/credentials.py"
               subPath: credentials.py
               readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
             - name: {{ ansible_operator_meta.name }}-settings
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
           volumes:
           - name: {{ ansible_operator_meta.name }}-metrics-utility
             persistentVolumeClaim:
@@ -71,10 +76,17 @@ spec:
               items:
                 - key: credentials.py
                   path: 'credentials.py'
+          - name: "{{ secret_key_secret_name }}"
+            secret:
+              secretName: '{{ secret_key_secret_name }}'
+              items:
+                - key: secret_key
+                  path: SECRET_KEY
           - name: {{ ansible_operator_meta.name }}-settings
             configMap:
               name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'
               items:
                 - key: settings
                   path: settings.py
+          {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=10) | trim }}
           restartPolicy: OnFailure

roles/installer/templates/deployments/task.yaml.j2

@@ -8,9 +8,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if task_replicas != '' %}
+{% if task_replicas != '' and task_manage_replicas | bool %}
   replicas: {{ task_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and task_manage_replicas | bool %}
   replicas: {{ replicas }}
 {% endif %}
   selector:
@@ -74,6 +74,25 @@ spec:
       priorityClassName: '{{ control_plane_priority_class }}'
 {% endif %}
       initContainers:
+{% if bundle_ca_crt  %}
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
         - name: init-database
           image: '{{ _image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -95,6 +114,11 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
+{% if bundle_ca_crt  %}
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+{% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
@@ -122,10 +146,6 @@ spec:
                 outcert=/etc/receptor/tls/receptor.crt \
                 notafter=$(date --iso-8601=seconds --utc --date "10 years") \
                 verify=yes
-{% if bundle_ca_crt  %}
-              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
-              update-ca-trust
-{% endif %}
 {% if init_container_extra_commands %}
               {{ init_container_extra_commands | indent(width=14) }}
 {% endif %}
@@ -148,10 +168,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if init_container_extra_volume_mounts -%}
             {{ init_container_extra_volume_mounts | indent(width=12, first=True) }}
@@ -244,7 +260,7 @@ spec:
             exec:
               command:
                 - /usr/bin/awx-manage
-                - check
+                - check_instance_ready
             initialDelaySeconds: {{ task_readiness_initial_delay }}
             periodSeconds: {{ task_readiness_period }}
             failureThreshold: {{ task_readiness_failure_threshold }}
@@ -254,10 +270,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: "{{ ansible_operator_meta.name }}-application-credentials"
               mountPath: "/etc/tower/conf.d/execution_environments.py"
@@ -279,6 +291,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -356,10 +369,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: "{{ ansible_operator_meta.name }}-default-receptor-config"
               mountPath: "/etc/receptor/receptor-default.conf"
@@ -428,6 +437,7 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -435,10 +445,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
@@ -588,6 +594,7 @@ spec:
             items:
               - key: redis_conf
                 path: redis.conf
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-redis-socket
           emptyDir: {}
         - name: {{ ansible_operator_meta.name }}-redis-data

roles/installer/templates/deployments/web.yaml.j2

@@ -9,9 +9,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if web_replicas != '' %}
+{% if web_replicas != '' and web_manage_replicas | bool %}
   replicas: {{ web_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and web_manage_replicas | bool %}
   replicas: {{ replicas }}
 {% endif %}
   selector:
@@ -80,7 +80,26 @@ spec:
       priorityClassName: '{{ control_plane_priority_class }}'
 {% endif %}
       initContainers:
-{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %}
+{% if bundle_ca_crt  %}
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
+{% if init_container_extra_commands %}
         - name: init
           image: '{{ _init_container_image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -89,21 +108,11 @@ spec:
             - /bin/sh
             - -c
             - |
-{% if bundle_ca_crt  %}
-              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
-              update-ca-trust
-{% endif %}
-{% if init_container_extra_commands %}
               {{ init_container_extra_commands | indent(width=14) }}
-{% endif %}
           volumeMounts:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if init_container_extra_volume_mounts -%}
             {{ init_container_extra_volume_mounts | indent(width=12, first=True) }}
@@ -191,10 +200,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: {{ ansible_operator_meta.name }}-uwsgi-config
               mountPath: "/etc/tower/uwsgi.ini"
@@ -231,6 +236,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-nginx-conf
               mountPath: /etc/nginx/nginx.conf
               subPath: nginx.conf
@@ -307,6 +313,7 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -314,10 +321,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
@@ -438,6 +441,7 @@ spec:
             items:
               - key: redis_conf
                 path: redis.conf
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-uwsgi-config
           configMap:
             name: {{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap

roles/installer/templates/jobs/migration.yaml.j2

@@ -8,7 +8,31 @@ metadata:
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
   template:
+    metadata:
+      labels:
+        {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=8) | trim }}
+        {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=8) | trim }}
     spec:
+{% if bundle_ca_crt %}
+      initContainers:
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
       containers:
         - name: "migration-job"
           image: '{{ _image }}'
@@ -29,6 +53,11 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
+{% if bundle_ca_crt %}
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+{% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
@@ -42,6 +71,34 @@ spec:
 {% for secret in image_pull_secrets %}
         - name: {{ secret }}
 {% endfor %}
+{% endif %}
+{% if task_node_selector %}
+      nodeSelector:
+        {{ task_node_selector | indent(width=8) }}
+{% elif node_selector %}
+      nodeSelector:
+        {{ node_selector | indent(width=8) }}
+{% endif %}
+{% if task_topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ task_topology_spread_constraints | indent(width=8) }}
+{% elif topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ topology_spread_constraints | indent(width=8) }}
+{% endif %}
+{% if task_tolerations %}
+      tolerations:
+        {{ task_tolerations | indent(width=8) }}
+{% elif tolerations %}
+      tolerations:
+        {{ tolerations | indent(width=8) }}
+{% endif %}
+{% if task_affinity %}
+      affinity:
+        {{ task_affinity | to_nice_yaml | indent(width=8) }}
+{% elif affinity %}
+      affinity:
+        {{ affinity | to_nice_yaml | indent(width=8) }}
 {% endif %}
       volumes:
         - name: "{{ ansible_operator_meta.name }}-application-credentials"
@@ -66,6 +123,17 @@ spec:
             items:
               - key: settings
                 path: settings.py
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
+{% if bundle_ca_crt %}
+        - name: "ca-trust-extracted"
+          emptyDir: {}
+        - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+          secret:
+            secretName: "{{ bundle_cacert_secret }}"
+            items:
+              - key: bundle-ca.crt
+                path: 'bundle-ca.crt'
+{% endif %}
 {% if development_mode | bool %}
         - name: awx-devel
           hostPath:

roles/installer/templates/settings/credentials.py.j2

@@ -10,6 +10,9 @@ DATABASES = {
         'OPTIONS': { 'sslmode': '{{ awx_postgres_sslmode }}',
 {% if awx_postgres_sslmode in ['verify-ca', 'verify-full'] %}
                      'sslrootcert': '{{ ca_trust_bundle }}',
+{% endif %}
+{% if awx_postgres_target_session_attrs %}
+                     'target_session_attrs': '{{ awx_postgres_target_session_attrs }}',
 {% endif %}
         },
     }
@@ -25,6 +28,9 @@ LISTENER_DATABASES = {
             'keepalives_count': {{ postgres_keepalives_count }},
 {% else %}
             'keepalives': 0,
+{% endif %}
+{% if awx_postgres_target_session_attrs %}
+            'target_session_attrs': '{{ awx_postgres_target_session_attrs }}',
 {% endif %}
         },
     }

roles/installer/vars/main.yml

@@ -8,3 +8,15 @@ supported_pg_version: 15
 _previous_upgraded_pg_version: 0
 old_postgres_pod: []
 _postgres_data_path: '/var/lib/pgsql/data/userdata'
+# metrics-utility (github.com/ansible/metrics-utility)
+_metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
+_metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
+_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
+_metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
+_metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
+_metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
+_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}"
+_metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
+_metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
+_metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"
+_metrics_utility_cronjob_report_schedule: "{{ metrics_utility_cronjob_report_schedule | default('@monthly') }}"

roles/mesh_ingress/defaults/main.yml

@@ -14,3 +14,8 @@ _image_pull_policy: Always
 image_pull_secrets: []
 
 finalizer_run: false
+
+node_selector: ''
+topology_spread_constraints: ''
+tolerations: ''
+affinity: {}

roles/mesh_ingress/templates/deployment.yml.j2

@@ -71,6 +71,22 @@ spec:
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccount: {{ ansible_operator_meta.name }}
+{% if node_selector %}
+      nodeSelector:
+        {{ node_selector | indent(width=8) }}
+{% endif %}
+{% if topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ topology_spread_constraints | indent(width=8) }}
+{% endif %}
+{% if tolerations %}
+      tolerations:
+        {{ tolerations | indent(width=8) }}
+{% endif %}
+{% if affinity %}
+      affinity:
+        {{ affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
       volumes:
       - name: {{ ansible_operator_meta.name }}-receptor-tls
       - name: {{ ansible_operator_meta.name }}-receptor-ca

roles/restore/defaults/main.yml

@@ -39,4 +39,6 @@ additional_labels: []
 
 # Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
 set_self_labels: true
+
+spec_overrides: {}
 ...

roles/restore/tasks/deploy_awx.yml

@@ -1,5 +1,10 @@
 ---
 
+- name: Combine spec_overrides with spec
+  set_fact:
+    spec: "{{ spec | default({}) | combine(spec_overrides) }}"
+  no_log: "{{ no_log }}"
+
 - name: Deploy AWX
   k8s:
     state: "{{ state | default('present') }}"

roles/restore/tasks/import_vars.yml

@@ -12,6 +12,7 @@
     k8s_cp:
       namespace: "{{ backup_pvc_namespace }}"
       pod: "{{ ansible_operator_meta.name }}-db-management"
+      container: "{{ ansible_operator_meta.name }}-db-management"
       remote_path: "{{ backup_dir  }}/awx_object"
       local_path: "{{ tmp_spec.path }}"
       state: from_pod

roles/restore/tasks/init.yml

@@ -86,10 +86,10 @@
   set_fact:
     _postgres_image: "{{ _custom_postgres_image | default(lookup('env', 'RELATED_IMAGE_AWX_POSTGRES'))  | default(_default_postgres_image, true) }}"
 
-- name: Create management pod from templated deployment config
+- name: Create management pod from the template
   k8s:
     name: "{{ ansible_operator_meta.name }}-db-management"
-    kind: Deployment
+    kind: Pod
     state: present
     definition: "{{ lookup('template', 'management-pod.yml.j2') }}"
     wait: true
@@ -98,6 +98,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       bash -c "stat {{ backup_dir }}"
   register: stat_backup_dir

roles/restore/tasks/postgres.yml

@@ -115,6 +115,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: |
       bash -c "
       function end_keepalive {

roles/restore/tasks/secrets.yml

@@ -10,6 +10,7 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir  }}/secrets.yml"
     local_path: "{{ tmp_secrets.path }}"
     state: from_pod