Wait for instance ready in molecule test (#1901)

Sometimes a job is launched through the web api
before the instance is in a ready state. This throws
a 500 internal server error, causing CI to fail.

Adds a task to query the instances endpoint
and check that at least one control node is
in a ready state.

Signed-off-by: Seth Foster <fosterbseth@gmail.com>

.github/workflows/reusable-nox.yml

@@ -18,7 +18,7 @@ jobs:
         - name: Check out repo
           uses: actions/checkout@v4
         - name: Setup nox
-          uses: wntrblm/nox@2024.03.02
+          uses: wntrblm/nox@2024.04.15
           with:
               python-versions: "${{ matrix.python-versions }}"
         - name: "Run nox -s ${{ matrix.session }}"

config/crd/bases/awx.ansible.com_awxmeshingresses.yaml

@@ -75,6 +75,381 @@ spec:
               ingress_controller:
                 description: Special configuration for specific Ingress Controllers
                 type: string
+              node_selector:
+                description: Assign the Mesh Ingress Pod to the specified node.
+                type: string
+              tolerations:
+                description: Scheduling tolerations for the Mesh Ingress instance.
+                type: string
+              topology_spread_constraints:
+                description: Topology spread constraints for the Mesh Ingress instance.
+                type: string
+              affinity:
+                description: Scheduling constraints to apply to the Pod definition
+                properties:
+                  nodeAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            preference:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - preference
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        properties:
+                          nodeSelectorTerms:
+                            items:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            type: array
+                        required:
+                        - nodeSelectorTerms
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                  podAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaceSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaceSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                  podAntiAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaceSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                  x-kubernetes-map-type: atomic
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaceSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                type: object
           status:
             description: Status defines the observed state of AWXMeshIngress
             type: object

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1574,10 +1574,18 @@ spec:
                 description: Number of web instance replicas
                 type: integer
                 format: int32
+              web_manage_replicas:
+                description: Enables operator control of replicas count for the web deployment when set to 'true'
+                type: boolean
+                default: true
               task_replicas:
                 description: Number of task instance replicas
                 type: integer
                 format: int32
+              task_manage_replicas:
+                description: Enables operator control of replicas count for the task deployment when set to 'true'
+                type: boolean
+                default: true
               web_liveness_initial_delay:
                 description: Initial delay before starting liveness checks on web pod
                 type: integer
@@ -1904,6 +1912,28 @@ spec:
                       x-kubernetes-preserve-unknown-fields: true
                   type: object
                 type: array
+              extra_settings_files:
+                description: Extra ConfigMaps or Secrets of settings files to specify for AWX
+                properties:
+                  configmaps:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                  secrets:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                type: object
               no_log:
                 description: Configure no_log for no_log tasks
                 type: boolean

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -58,6 +58,26 @@ spec:
         path: ingress_controller
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Node Selector
+        path: node_selector
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Tolerations
+        path: tolerations
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Topology Spread Constraints
+        path: topology_spread_constraints
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Affinity
+        path: affinity
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Optional API URLPATTERN Prefix
         path: api_urlpattern_prefix
         x-descriptors:
@@ -966,6 +986,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Extra Settings Files
+        path: extra_settings_files
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: No Log Configuration
         path: no_log
         x-descriptors:

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -1,4 +1,4 @@
-#### Custom Volume and Volume Mount Options
+# Custom Volume and Volume Mount Options
 
 In a scenario where custom volumes and volume mounts are required to either overwrite defaults or mount configuration files.
 
@@ -12,7 +12,6 @@ In a scenario where custom volumes and volume mounts are required to either over
 | init_container_extra_volume_mounts | Specify volume mounts to be added to Init container      | ''      |
 | init_container_extra_commands      | Specify additional commands for Init container           | ''      |
 
-
 !!! warning
     The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
 
@@ -31,10 +30,8 @@ data:
     remote_tmp = /tmp
     [ssh_connection]
     ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
-  custom.py:  |
-    INSIGHTS_URL_BASE = "example.org"
-    AWX_CLEANUP_PATHS = True
 ```
+
 Example spec file for volumes and volume mounts
 
 ```yaml
@@ -49,13 +46,6 @@ spec:
           - key: ansible.cfg
             path: ansible.cfg
         name: <resourcename>-extra-config
-    - name: custom-py
-      configMap:
-        defaultMode: 420
-        items:
-          - key: custom.py
-            path: custom.py
-        name: <resourcename>-extra-config
     - name: shared-volume
       persistentVolumeClaim:
         claimName: my-external-volume-claim
@@ -73,24 +63,13 @@ spec:
     - name: ansible-cfg
       mountPath: /etc/ansible/ansible.cfg
       subPath: ansible.cfg
-
-  web_extra_volume_mounts: |
-    - name: custom-py
-      mountPath: /etc/tower/conf.d/custom.py
-      subPath: custom.py
-
-  task_extra_volume_mounts: |
-    - name: custom-py
-      mountPath: /etc/tower/conf.d/custom.py
-      subPath: custom.py
-    - name: shared-volume
-      mountPath: /shared
 ```
 
 !!! warning
     **Volume and VolumeMount names cannot contain underscores(_)**
 
-##### Custom UWSGI Configuration
+## Custom UWSGI Configuration
+
 We allow the customization of two UWSGI parameters:
 
 * [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5)
@@ -110,7 +89,7 @@ requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related nginx
 configuration.
 
-##### Custom Nginx Configuration
+## Custom Nginx Configuration
 
 Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf.
 
@@ -131,13 +110,13 @@ may allow the web pods to handle more "bursty" request patterns if many
 requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related uwsgi
 configuration.
+
 * [worker_processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes) with `nginx_worker_processes` (default of 1)
 * [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) with `nginx_worker_cpu_affinity` (default "auto")
 * [worker_connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) with `nginx_worker_connections` (minimum of 1024)
 * [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size)
 
-
-##### Custom Logos
+## Custom Logos
 
 You can use custom volume mounts to mount in your own logos to be displayed instead of the AWX logo.
 There are two different logos, one to be displayed on page headers, and one for the login screen.
@@ -145,8 +124,8 @@ There are two different logos, one to be displayed on page headers, and one for
 First, create configmaps for the logos from local `logo-login.svg` and `logo-header.svg` files.
 
 ```bash
-$ kubectl create configmap logo-login-configmap --from-file logo-login.svg
-$ kubectl create configmap logo-header-configmap --from-file logo-header.svg
+kubectl create configmap logo-login-configmap --from-file logo-login.svg
+kubectl create configmap logo-header-configmap --from-file logo-header.svg
 ```
 
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
@@ -179,15 +158,14 @@ spec:
       subPath: logo-header.svg
 ```
 
-
-##### Custom Favicon
+## Custom Favicon
 
 You can also use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
 
 First, create the configmap from a local `favicon.ico` file.
 
 ```bash
-$ kubectl create configmap favicon-configmap --from-file favicon.ico
+kubectl create configmap favicon-configmap --from-file favicon.ico
 ```
 
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
@@ -209,3 +187,7 @@ spec:
       mountPath: /var/lib/awx/public/static/media/favicon.ico
       subPath: favicon.ico
 ```
+
+## Custom AWX Configuration
+
+Refer to the [Extra Settings](./extra-settings.md) documentation for customizing the AWX configuration.

docs/user-guide/advanced-configuration/extra-settings.md

@@ -1,30 +1,119 @@
-#### Extra Settings
+# Extra Settings
 
-With`extra_settings`, you can pass multiple custom settings via the `awx-operator`. The parameter `extra_settings`  will be appended to the `/etc/tower/settings.py` and can be an alternative to the `extra_volumes` parameter.
+With `extra_settings` and `extra_settings_files`, you can pass multiple custom settings to AWX via the AWX Operator.
 
-| Name           | Description    | Default |
-| -------------- | -------------- | ------- |
-| extra_settings | Extra settings | ''      |
+!!! note
+    Parameters configured in `extra_settings` or `extra_settings_files` are set as read-only settings in AWX. As a result, they cannot be changed in the UI after deployment.
 
-**Note:** Parameters configured in `extra_settings` are set as read-only settings in AWX.  As a result, they cannot be changed in the UI after deployment. If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec.  
+    If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec (for `extra_settings`) or corresponding ConfigMap or Secret (for `extra_settings_files`). After updating ConfigMap or Secret, you need to restart the AWX pods to apply the changes.
+
+!!! note
+    If the same setting is set in both `extra_settings` and `extra_settings_files`, the setting in `extra_settings_files` will take precedence.
+
+## Add extra settings with `extra_settings`
+
+You can pass extra settings by specifying the pair of the setting name and value as the `extra_settings` parameter.
+
+The settings passed via `extra_settings` will be appended to the `/etc/tower/settings.py`.
+
+| Name           | Description    | Default   |
+| -------------- | -------------- | --------- |
+| extra_settings | Extra settings | `[]`      |
 
 Example configuration of `extra_settings` parameter
 
 ```yaml
-  spec:
-    extra_settings:
-      - setting: MAX_PAGE_SIZE
-        value: "500"
+spec:
+  extra_settings:
+    - setting: MAX_PAGE_SIZE
+      value: "500"
 
-      - setting: AUTH_LDAP_BIND_DN
-        value: "cn=admin,dc=example,dc=com"
+    - setting: AUTH_LDAP_BIND_DN
+      value: "cn=admin,dc=example,dc=com"
 
-      - setting: LOG_AGGREGATOR_LEVEL
-        value: "'DEBUG'"
+    - setting: LOG_AGGREGATOR_LEVEL
+      value: "'DEBUG'"
 ```
 
 Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes.
 
-!!! tip
-    Alternatively, you can pass any additional settings by mounting ConfigMaps or Secrets of the python files (`*.py`) that contain custom settings to under `/etc/tower/conf.d/` in the web and task pods.
-    See the example of `custom.py` in the [Custom Volume and Volume Mount Options](custom-volume-and-volume-mount-options.md) section.
+## Add extra settings with `extra_settings_files`
+
+You can pass extra settings by specifying the additional settings files in the ConfigMaps or Secrets as the `extra_settings_files` parameter.
+
+The settings files passed via `extra_settings_files` will be mounted as the files under the `/etc/tower/conf.d`.
+
+| Name                 | Description          | Default   |
+| -------------------- | -------------------- | --------- |
+| extra_settings_files | Extra settings files | `{}`      |
+
+!!! note
+    If the same setting is set in multiple files in `extra_settings_files`, it would be difficult to predict which would be adopted since these files are loaded in arbitrary order that [`glob`](https://docs.python.org/3/library/glob.html) returns. For a reliable setting, do not include the same key in more than one file.
+
+Create ConfigMaps or Secrets that contain custom settings files (`*.py`).
+
+```python title="custom_job_settings.py"
+AWX_TASK_ENV = {
+    "HTTPS_PROXY": "http://proxy.example.com:3128",
+    "HTTP_PROXY": "http://proxy.example.com:3128",
+    "NO_PROXY": "127.0.0.1,localhost,.example.com"
+}
+GALAXY_TASK_ENV = {
+    "ANSIBLE_FORCE_COLOR": "false",
+    "GIT_SSH_COMMAND": "ssh -o StrictHostKeyChecking=no",
+}
+```
+
+```python title="custom_system_settings.py"
+REMOTE_HOST_HEADERS = [
+    "HTTP_X_FORWARDED_FOR",
+    "REMOTE_ADDR",
+    "REMOTE_HOST",
+]
+```
+
+```python title="custom_passwords.py"
+SUBSCRIPTIONS_PASSWORD = "my-super-secure-subscription-password123!"
+REDHAT_PASSWORD = "my-super-secure-redhat-password123!"
+```
+
+```bash title="Create ConfigMap and Secret"
+# Create ConfigMap
+kubectl create configmap my-custom-settings \
+  --from-file /PATH/TO/YOUR/custom_job_settings.py \
+  --from-file /PATH/TO/YOUR/custom_system_settings.py
+
+# Create Secret
+kubectl create secret generic my-custom-passwords \
+  --from-file /PATH/TO/YOUR/custom_passwords.py
+```
+
+Then specify them in the AWX CR spec. Here is an example configuration of `extra_settings_files` parameter.
+
+```yaml
+spec:
+  extra_settings_files:
+    configmaps:
+      - name: my-custom-settings        # The name of the ConfigMap
+        key: custom_job_settings.py     # The key in the ConfigMap, which means the file name
+      - name: my-custom-settings
+        key: custom_system_settings.py
+    secrets:
+      - name: my-custom-passwords       # The name of the Secret
+        key: custom_passwords.py        # The key in the Secret, which means the file name
+```
+
+!!! Warning "Restriction"
+    There are some restrictions on the ConfigMaps or Secrets used in `extra_settings_files`.
+
+    - The keys in ConfigMaps or Secrets MUST be the name of python files and MUST end with `.py`
+    - The keys in ConfigMaps or Secrets MUST consists of alphanumeric characters, `-`, `_` or `.`
+    - The keys in ConfigMaps or Secrets are converted to the following strings, which MUST not exceed 63 characters
+        - Keys in ConfigMaps: `<instance name>-<KEY>-configmap`
+        - Keys in Secrets: `<instance name>-<KEY>-secret`
+    - Following keys are reserved and MUST NOT be used in ConfigMaps or Secrets
+        - `credentials.py`
+        - `execution_environments.py`
+        - `ldap.py`
+
+    Refer to the Kubernetes documentations ([[1]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/config-map-v1/), [[2]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/), [[3]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/), [[4]](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)) for more information about character types and length restrictions.

docs/user-guide/advanced-configuration/horizontal-pod-autoscaler.md

@@ -0,0 +1,27 @@
+### Horizontal Pod Autoscaler (HPA)
+
+Horizontal Pod Autoscaler allows Kubernetes to scale the number of replicas of
+deployments in response to configured metrics.
+
+This feature conflicts with the operators ability to manage the number of static
+replicas to create for each deployment.
+
+The use of the settings below will tell the operator to not manage the replicas
+field on the identified deployments even if a replicas count has been set for those
+properties in the operator resource.
+
+| Name                   | Description                               | Default |
+| -----------------------| ----------------------------------------- | ------- |
+| web_manage_replicas    | Indicates operator should control the     | true    |
+|                        | replicas count for the web deployment.    |         |
+|                        |                                           |         |
+| task_manage_replicas   | Indicates operator should control the     | true    |
+|                        | replicas count for the task deployment.   |         |
+
+#### Recommended Settings for HPA
+
+Please see the Kubernetes documentation on how to configure the horizontal pod
+autoscaler.
+
+The values for optimal HPA are cluster and need specific so general guidelines
+are not available at this time.

docs/user-guide/advanced-configuration/scaling-the-web-and-task-pods-independently.md

@@ -1,8 +1,13 @@
-#### Scaling the Web and Task Pods independently 
+#### Scaling the Web and Task Pods independently
 
 You can scale replicas up or down for each deployment by using the `web_replicas` or `task_replicas` respectively. You can scale all pods across both deployments by using `replicas` as well. The logic behind these CRD keys acts as such:
 
-- If you specify the `replicas` field, the key passed will scale both the `web` and `task` replicas to the same number. 
+- If you specify the `replicas` field, the key passed will scale both the `web` and `task` replicas to the same number.
 - If `web_replicas` or `task_replicas` is ever passed, it will override the existing `replicas` field on the specific deployment with the new key value.
 
-These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found in the [Assigning AWX pods to specific nodes](./assigning-awx-pods-to-specific-nodes.md) page. 
+These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found in the [Assigning AWX pods to specific nodes](./assigning-awx-pods-to-specific-nodes.md) page.
+
+##### Horizontal Pod Autoscaling
+
+The operator is capable of working with Kubernete's HPA capabilities.  See [Horizontal Pod Autoscaler](./horizontal-pod-autoscaler.md)
+documentation for more information.

molecule/default/tasks/awx_test.yml

@@ -127,6 +127,17 @@
     name: example-awx-admin-password
   register: admin_pw_secret
 
+- name: Wait for instance to be ready
+  uri:
+    url: "http://localhost/awx/api/v2/instances/?node_type=control&node_state=ready"
+    user: admin
+    password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+    force_basic_auth: yes
+  register: instances
+  until: instances['json']['count'] | int > 0
+  retries: 20
+  delay: 2
+
 - name: Validate demo job launch
   block:
     - name: Launch Demo Job Template

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -45,6 +45,7 @@ spec:
   extra_settings:
   - setting: LOG_AGGREGATOR_LEVEL
     value: "'DEBUG'"
+  task_readiness_period: 15
 {% if additional_fields is defined %}
   {{ additional_fields | to_nice_yaml | indent(2) }}
 {% endif %}

roles/installer/defaults/main.yml

@@ -491,3 +491,5 @@ nginx_worker_processes: 1
 nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
 nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
+
+extra_settings_files: {}

roles/installer/tasks/migrate_schema.yml

@@ -24,7 +24,7 @@
 
     - name: Sanitize instance version
       set_fact:
-        version: "{{ version_check.stdout.split('+')[0] | trim }}"
+        version: "{{ version_check.stdout | replace('+', '-') | trim }}"
 
     # It is possible to do a wait on this task to create the job and wait
     # until it completes. Unfortunately, if the job doesn't wait finish within

roles/installer/templates/common/volume_mounts/extra_settings_files.yaml.j2

@@ -0,0 +1,16 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  mountPath: "/etc/tower/conf.d/{{ configmap.key }}"
+  subPath: {{ configmap.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  mountPath: "/etc/tower/conf.d/{{ secret.key }}"
+  subPath: {{ secret.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}

roles/installer/templates/common/volumes/extra_settings_files.yaml.j2

@@ -0,0 +1,20 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  configMap:
+    name: {{ configmap.name }}
+    items:
+      - key: {{ configmap.key }}
+        path: {{ configmap.key }}
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  secret:
+    secretName: {{ secret.name }}
+    items:
+      - key: {{ secret.key }}
+        path: {{ secret.key }}
+{% endfor %}
+{% endif %}

roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2

@@ -47,9 +47,9 @@ spec:
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
-{% if _metrics_utility_secret is defined %}
+{% if metrics_utility_secret is defined %}
             - secretRef:
-                name: {{ _metrics_utility_secret }}
+                name: {{ metrics_utility_secret }}
 {% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
@@ -67,6 +67,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
           volumes:
           - name: {{ ansible_operator_meta.name }}-metrics-utility
             persistentVolumeClaim:
@@ -90,4 +91,5 @@ spec:
               items:
                 - key: settings
                   path: settings.py
+          {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=10) | trim }}
           restartPolicy: OnFailure

roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2

@@ -44,9 +44,9 @@ spec:
             envFrom:
             - configMapRef:
                 name: {{ _metrics_utility_configmap }}
-{% if _metrics_utility_secret is defined %}
+{% if metrics_utility_secret is defined %}
             - secretRef:
-                name: {{ _metrics_utility_secret }}
+                name: {{ metrics_utility_secret }}
 {% endif %}
             volumeMounts:
             - name: {{ ansible_operator_meta.name }}-metrics-utility
@@ -64,6 +64,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
           volumes:
           - name: {{ ansible_operator_meta.name }}-metrics-utility
             persistentVolumeClaim:
@@ -87,4 +88,5 @@ spec:
               items:
                 - key: settings
                   path: settings.py
+          {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=10) | trim }}
           restartPolicy: OnFailure

roles/installer/templates/deployments/task.yaml.j2

@@ -8,9 +8,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if task_replicas != '' %}
+{% if task_replicas != '' and task_manage_replicas | bool %}
   replicas: {{ task_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and task_manage_replicas | bool %}
   replicas: {{ replicas }}
 {% endif %}
   selector:
@@ -74,6 +74,25 @@ spec:
       priorityClassName: '{{ control_plane_priority_class }}'
 {% endif %}
       initContainers:
+{% if bundle_ca_crt  %}
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
         - name: init-database
           image: '{{ _image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -95,6 +114,11 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
+{% if bundle_ca_crt  %}
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+{% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
@@ -122,10 +146,6 @@ spec:
                 outcert=/etc/receptor/tls/receptor.crt \
                 notafter=$(date --iso-8601=seconds --utc --date "10 years") \
                 verify=yes
-{% if bundle_ca_crt  %}
-              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
-              update-ca-trust
-{% endif %}
 {% if init_container_extra_commands %}
               {{ init_container_extra_commands | indent(width=14) }}
 {% endif %}
@@ -148,10 +168,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if init_container_extra_volume_mounts -%}
             {{ init_container_extra_volume_mounts | indent(width=12, first=True) }}
@@ -244,7 +260,7 @@ spec:
             exec:
               command:
                 - /usr/bin/awx-manage
-                - check
+                - check_instance_ready
             initialDelaySeconds: {{ task_readiness_initial_delay }}
             periodSeconds: {{ task_readiness_period }}
             failureThreshold: {{ task_readiness_failure_threshold }}
@@ -254,10 +270,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: "{{ ansible_operator_meta.name }}-application-credentials"
               mountPath: "/etc/tower/conf.d/execution_environments.py"
@@ -279,6 +291,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -356,10 +369,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: "{{ ansible_operator_meta.name }}-default-receptor-config"
               mountPath: "/etc/receptor/receptor-default.conf"
@@ -428,6 +437,7 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -435,10 +445,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
@@ -588,6 +594,7 @@ spec:
             items:
               - key: redis_conf
                 path: redis.conf
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-redis-socket
           emptyDir: {}
         - name: {{ ansible_operator_meta.name }}-redis-data

roles/installer/templates/deployments/web.yaml.j2

@@ -9,9 +9,9 @@ metadata:
     {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=4) | trim }}
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
-{% if web_replicas != '' %}
+{% if web_replicas != '' and web_manage_replicas | bool %}
   replicas: {{ web_replicas }}
-{% elif replicas != '' %}
+{% elif replicas != '' and web_manage_replicas | bool %}
   replicas: {{ replicas }}
 {% endif %}
   selector:
@@ -80,7 +80,26 @@ spec:
       priorityClassName: '{{ control_plane_priority_class }}'
 {% endif %}
       initContainers:
-{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %}
+{% if bundle_ca_crt  %}
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
+{% if init_container_extra_commands %}
         - name: init
           image: '{{ _init_container_image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -89,21 +108,11 @@ spec:
             - /bin/sh
             - -c
             - |
-{% if bundle_ca_crt  %}
-              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
-              update-ca-trust
-{% endif %}
-{% if init_container_extra_commands %}
               {{ init_container_extra_commands | indent(width=14) }}
-{% endif %}
           volumeMounts:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if init_container_extra_volume_mounts -%}
             {{ init_container_extra_volume_mounts | indent(width=12, first=True) }}
@@ -191,10 +200,6 @@ spec:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
             - name: {{ ansible_operator_meta.name }}-uwsgi-config
               mountPath: "/etc/tower/uwsgi.ini"
@@ -231,6 +236,7 @@ spec:
               mountPath: /etc/tower/settings.py
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-nginx-conf
               mountPath: /etc/nginx/nginx.conf
               subPath: nginx.conf
@@ -307,6 +313,7 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-redis-socket
               mountPath: "/var/run/redis"
             - name: rsyslog-socket
@@ -314,10 +321,6 @@ spec:
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
-            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
-              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
-              subPath: bundle-ca.crt
-              readOnly: true
 {% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
@@ -438,6 +441,7 @@ spec:
             items:
               - key: redis_conf
                 path: redis.conf
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-uwsgi-config
           configMap:
             name: {{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap

roles/installer/templates/jobs/migration.yaml.j2

@@ -8,7 +8,31 @@ metadata:
     {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
 spec:
   template:
+    metadata:
+      labels:
+        {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=8) | trim }}
+        {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=8) | trim }}
     spec:
+{% if bundle_ca_crt %}
+      initContainers:
+        - name: init-bundle-ca-trust
+          image: '{{ _init_container_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          resources: {{ init_container_resource_requirements }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
+              update-ca-trust extract
+          volumeMounts:
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+            - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+              mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt
+              subPath: bundle-ca.crt
+              readOnly: true
+{% endif %}
       containers:
         - name: "migration-job"
           image: '{{ _image }}'
@@ -29,6 +53,11 @@ spec:
               mountPath: "/etc/tower/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
+{% if bundle_ca_crt %}
+            - name: "ca-trust-extracted"
+              mountPath: "/etc/pki/ca-trust/extracted"
+{% endif %}
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
@@ -94,6 +123,17 @@ spec:
             items:
               - key: settings
                 path: settings.py
+        {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
+{% if bundle_ca_crt %}
+        - name: "ca-trust-extracted"
+          emptyDir: {}
+        - name: "{{ ansible_operator_meta.name }}-bundle-cacert"
+          secret:
+            secretName: "{{ bundle_cacert_secret }}"
+            items:
+              - key: bundle-ca.crt
+                path: 'bundle-ca.crt'
+{% endif %}
 {% if development_mode | bool %}
         - name: awx-devel
           hostPath:

roles/installer/vars/main.yml

@@ -11,7 +11,6 @@ _postgres_data_path: '/var/lib/pgsql/data/userdata'
 # metrics-utility (github.com/ansible/metrics-utility)
 _metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
 _metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
-_metrics_utility_secret: "{{ metrics_utility_secret | default('') }}"
 _metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}"
 _metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
 _metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"

roles/mesh_ingress/defaults/main.yml

@@ -14,3 +14,8 @@ _image_pull_policy: Always
 image_pull_secrets: []
 
 finalizer_run: false
+
+node_selector: ''
+topology_spread_constraints: ''
+tolerations: ''
+affinity: {}

roles/mesh_ingress/templates/deployment.yml.j2

@@ -71,6 +71,22 @@ spec:
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccount: {{ ansible_operator_meta.name }}
+{% if node_selector %}
+      nodeSelector:
+        {{ node_selector | indent(width=8) }}
+{% endif %}
+{% if topology_spread_constraints %}
+      topologySpreadConstraints:
+        {{ topology_spread_constraints | indent(width=8) }}
+{% endif %}
+{% if tolerations %}
+      tolerations:
+        {{ tolerations | indent(width=8) }}
+{% endif %}
+{% if affinity %}
+      affinity:
+        {{ affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
       volumes:
       - name: {{ ansible_operator_meta.name }}-receptor-tls
       - name: {{ ansible_operator_meta.name }}-receptor-ca