Set default for -iness probe parameters and add docs (#1808)

Add postgres init container if
postgres_data_volume_init is true

This is aimed to solve the issue where users may
need to chmod or chown the postgres
data volume for user 26, which is the user
that is running postgres in the sclorg image.

For example, one can now set the follow on the AWX spec:
spec:
  postgres_init_container_commands: |
    chown 26:0 /var/lib/pgsql/data
    chmod 700 /var/lib/pgsql/data

Deprecate postgres_init_container_resource_requirements param in favor
of postgres_resource_requirements param.

Signed-off-by: Seth Foster <fosterbseth@gmail.com>
Co-authored-by: craph <14820052+craph@users.noreply.github.com>
Co-authored-by: kurokobo <kuro664@gmail.com>
Co-authored-by: Christian M. Adams <chadams@redhat.com>

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/docs"
+    groups:
+      dependencies:
+        patterns:
+          - "*"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "component:docs"
+      - "dependencies"

.github/workflows/ci.yaml

@@ -99,3 +99,5 @@ jobs:
             echo "${no_log}"
             exit 1
           fi
+  nox-sessions:
+    uses: ./.github/workflows/reusable-nox.yml

.github/workflows/reusable-nox.yml

@@ -0,0 +1,26 @@
+---
+name: nox
+
+"on":
+    workflow_call:
+
+jobs:
+    nox:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                - session: build
+                  python-versions: "3.11"
+        name: "Run nox ${{ matrix.session }} session"
+        steps:
+        - name: Check out repo
+          uses: actions/checkout@v4
+        - name: Setup nox
+          uses: wntrblm/nox@2024.03.02
+          with:
+              python-versions: "${{ matrix.python-versions }}"
+        - name: "Run nox -s ${{ matrix.session }}"
+          run: |
+            nox -s "${{ matrix.session }}"

.helm/starter/templates/storage/postgres-pv.yaml

@@ -13,7 +13,7 @@ spec:
     storage: {{ default "8Gi" .size | quote }}
   storageClassName: {{ include "postgres.storageClassName" $ }}
   hostPath:
-    path: {{ required "customVolumes.postgres.hostPath or spec.postgres_data_path are required!" (default ($.Values.AWX.spec).postgres_data_path .hostPath) | quote }}
+    path: /var/lib/pgsql/data/userdata
 {{- end }}
 {{- end }}
 {{- end }}

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1460,7 +1460,7 @@ spec:
                     type: object
                 type: object
               postgres_init_container_resource_requirements:
-                description: Resource requirements for the postgres init container
+                description: (Deprecated, use postgres_resource_requirements parameter) Resource requirements for the postgres init container
                 properties:
                   requests:
                     properties:
@@ -1807,13 +1807,16 @@ spec:
               postgres_priority_class:
                 description: Assign a preexisting priority class to the postgres pod
                 type: string
-              postgres_data_path:
-                description: Path where the PostgreSQL data are located
-                type: string
               postgres_extra_args:
                 type: array
                 items:
                   type: string
+              postgres_data_volume_init:
+                description: Sets permissions on the /var/lib/pgdata/data for postgres container using an init container (not Openshift)
+                type: boolean
+              postgres_init_container_commands:
+                description: Customize the postgres init container commands (Non Openshift)
+                type: string
               postgres_extra_volumes:
                 description: Specify extra volumes to add to the application pod
                 type: string
@@ -1923,6 +1926,40 @@ spec:
                 description: Disable web container's nginx ipv6 listener
                 type: boolean
                 default: false
+              metrics_utility_enabled:
+                description: Enable metrics utility
+                type: boolean
+                default: false
+              metrics_utility_image:
+                description: Metrics-Utility Image
+                type: string
+              metrics_utility_image_version:
+                description: Metrics-Utility Image Version
+                type: string
+              metrics_utility_image_pull_policy:
+                description: Metrics-Utility Image PullPolicy
+                type: string
+              metrics_utility_configmap:
+                description: Metrics-Utlity ConfigMap
+                type: string
+              metrics_utility_cronjob_gather_schedule:
+                description: Metrics-Utlity Gather Data CronJob Schedule
+                type: string
+                default: '@hourly'
+              metrics_utility_cronjob_report_schedule:
+                description: Metrics-Utlity Report CronJob Schedule
+                type: string
+                default: '@monthly'
+              metrics_utility_pvc_claim:
+                description: Metrics-Utlity PVC Claim
+                type: string
+              metrics_utility_pvc_claim_size:
+                description: Metrics-Utlity PVC Claim Size
+                type: string
+                default: 5Gi
+              metrics_utility_pvc_claim_storage_class:
+                description: Metrics-Utlity PVC Claim Storage Class
+                type: string
             type: object
           status:
             properties:

config/manager/manager.yaml

@@ -39,7 +39,7 @@ spec:
         - --leader-elect
         - --leader-election-id=awx-operator
         image: controller:latest
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: awx-manager
         env:
         - name: ANSIBLE_GATHERING

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -456,12 +456,21 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
-      - description: The PostgreSQL init container is not used when an external DB
+      - description: Sets permissions on the /var/lib/pgsql/data for postgres container using an init container (not Openshift)
-          is configured
+        displayName: PostgreSQL initialize data volume
+        path: postgres_data_volume_init
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: Customize the postgres init container commands (Non Openshift)
+        displayName: PostgreSQL Init Container Commands
+        path: postgres_init_container_commands
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: (Deprecated, use postgres_resource_requirements parameter instead)
         displayName: PostgreSQL Init Container Resource Requirements
         path: postgres_init_container_resource_requirements
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
       - displayName: Redis Container Resource Requirements
         path: redis_resource_requirements
@@ -637,11 +646,6 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:advanced
-      - displayName: Postgres Datapath
-        path: postgres_data_path
-        x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
-        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Postgres Extra Arguments
         path: postgres_extra_args
         x-descriptors:
@@ -1024,6 +1028,65 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - displayName: Metrics-Utility Enabled
+        path: metrics_utility_enabled
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - displayName: Metrics-Utility Image
+        path: metrics_utility_image
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utlity Image Version
+        path: metrics_utility_image_version
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Image PullPolicy
+        path: metrics_utility_image_pull_policy
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:imagePullPolicy
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utlity ConfigMap
+        path: metrics_utility_configmap
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:ConfigMap
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utlity Gather Data CronJob Schedule
+        path: metrics_utility_cronjob_gather_schedule
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utlity Report CronJob Schedule
+        path: metrics_utility_cronjob_report_schedule
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utlity PVC Claim
+        path: metrics_utility_pvc_claim
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utlity PVC Claim Size
+        path: metrics_utility_pvc_claim_size
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utlity PVC Claim Storage Class
+        path: metrics_utility_pvc_claim_storage_class
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:StorageClass
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
       statusDescriptors:
       - description: Route to access the instance deployed
         displayName: URL

config/rbac/role.yaml

@@ -81,6 +81,7 @@ rules:
   - apiGroups:
       - batch
     resources:
+      - cronjobs
       - jobs
     verbs:
       - get

config/samples/awx_v1beta1_awx_resource_limits.yaml

@@ -46,10 +46,3 @@ spec:
     limits:
       cpu: 1000m
       memory: 2Gi
-  postgres_init_container_resource_requirements:
-    requests:
-      cpu: 10m
-      memory: 64Mi
-    limits:
-      cpu: 1000m
-      memory: 2Gi

docs/user-guide/advanced-configuration/container-probes.md

@@ -2,6 +2,17 @@
 These parameters control the usage of liveness and readiness container probes for
 the web and task containers.
 
+> [!ALERT]
+> All of probes are disabled by default for now, to enable it, set the *_period parameters.  For example:
+
+```
+
+web_liveness_period: 15
+web_readiness_period: 15
+task_liveness_period: 15
+task_readiness_period: 15
+```
+
 #### Web / Task Container Liveness Check
 
 The liveness probe queries the status of the supervisor daemon of the container.  The probe will fail if it

docs/user-guide/advanced-configuration/containers-resource-requirements.md

@@ -22,16 +22,15 @@ spec:
 
 The resource requirements for both, the task and the web containers are configurable - both the lower end (requests) and the upper end (limits).
 
-| Name                       | Description                                      | Default                              |
+| Name                                 | Description                                                  | Default                              |
-| -------------------------- | ------------------------------------------------ | ------------------------------------ |
+| ------------------------------------ | ------------------------------------------------------------ | ------------------------------------ |
-| web_resource_requirements  | Web container resource requirements              | requests: {cpu: 100m, memory: 128Mi} |
+| web_resource_requirements            | Web container resource requirements                          | requests: {cpu: 100m, memory: 128Mi} |
-| task_resource_requirements | Task container resource requirements             | requests: {cpu: 100m, memory: 128Mi} |
+| task_resource_requirements           | Task container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
-| ee_resource_requirements   | EE control plane container resource requirements | requests: {cpu: 50m, memory: 64Mi} |
+| ee_resource_requirements             | EE control plane container resource requirements             | requests: {cpu: 50m, memory: 64Mi} |
-| redis_resource_requirements   | Redis container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
+| redis_resource_requirements          | Redis container resource requirements                        | requests: {cpu: 100m, memory: 128Mi} |
-| postgres_resource_requirements   | Postgres container resource requirements | requests: {cpu: 10m, memory: 64Mi} |
+| postgres_resource_requirements       | Postgres container (and initContainer) resource requirements | requests: {cpu: 10m, memory: 64Mi}   |
-| rsyslog_resource_requirements   | Rsyslog container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
+| rsyslog_resource_requirements        | Rsyslog container resource requirements                      | requests: {cpu: 100m, memory: 128Mi} |
-| init_container_resource_requirements   | Init Container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
+| init_container_resource_requirements | Init Container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
-| postgres_init_container_resource_requirements   | Postgres Init Container resource requirements | requests: {cpu: 10m, memory: 64Mi} |
 
 
 Example of customization could be:
@@ -85,13 +84,6 @@ spec:
     limits:
       cpu: 1000m
       memory: 2Gi
-  postgres_init_container_resource_requirements:
-    requests:
-      cpu: 10m
-      memory: 64Mi
-    limits:
-      cpu: 1000m
-      memory: 2Gi
 ```
 
 

docs/user-guide/advanced-configuration/deploying-a-specific-version-of-awx.md

@@ -2,15 +2,19 @@
 
 There are a few variables that are customizable for awx the image management.
 
-| Name                | Description               | Default                                    |
+| Name                          | Description               | Default                                    |
-| ------------------- | ------------------------- | -----------------------------------------  |
+| ----------------------------- | ------------------------- | ------------------------------------------ |
-| image               | Path of the image to pull | quay.io/ansible/awx                        |
+| image                         | Path of the image to pull | quay.io/ansible/awx                        |
-| image_version       | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| image_version                 | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
-| image_pull_policy   | The pull policy to adopt  | IfNotPresent                               |
+| image_pull_policy             | The pull policy to adopt  | IfNotPresent                               |
-| image_pull_secrets  | The pull secrets to use   | None                                       |
+| image_pull_secrets            | The pull secrets to use   | None                                       |
-| ee_images           | A list of EEs to register | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| ee_images                     | A list of EEs to register | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
-| redis_image         | Path of the image to pull | docker.io/redis                            |
+| redis_image                   | Path of the image to pull | docker.io/redis                            |
-| redis_image_version | Image version to pull     | latest                                     |
+| redis_image_version           | Image version to pull     | latest                                     |
+| control_plane_ee_image        | Image version to pull     | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| init_container_image          | Path of the image to pull | quay.io/ansible/awx-ee                     |
+| init_container_image_version  | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| init_projects_container_image | Image version to pull     | quay.io/centos/centos:stream9              |
 
 Example of customization could be:
 
@@ -26,6 +30,10 @@ spec:
   ee_images:
     - name: my-custom-awx-ee
       image: myorg/my-custom-awx-ee
+  control_plane_ee_image: myorg/my-custom-awx-ee:latest
+  init_container_image: myorg/my-custom-awx-ee
+  init_container_image_version: latest
+  init_projects_container_image: myorg/my-mirrored-centos:stream9
 ```
 
 **Note**: The `image` and `image_version` are intended for local mirroring scenarios. Please note that using a version of AWX other than the one bundled with the `awx-operator` is **not** supported. For the default values, check the [main.yml](https://github.com/ansible/awx-operator/blob/devel/roles/installer/defaults/main.yml) file.

docs/user-guide/advanced-configuration/mesh-ingress.md

@@ -83,6 +83,11 @@ spec:
   external_hostname: <fqdn for mesh ingress>
 ```
 
+!!! tip
+    AWX Operator supports both API groups `traefik.io` and `traefik.containo.us` in `ingress_api_version` for Traefik, but it is recommended to use `traefik.io` since `traefik.containo.us` is deprecated in Traefik v2.10 and is removed in Traefik v3. Refer to [Traefik documentation](https://doc.traefik.io/traefik/migration/v2/#v210) for more information about deprecation.
+
+    If you can't see any IngressRouteTCP resources by `kubectl` command after deploying mesh ingress, you should fully qualify the resource name with API group, `kubectl get ingressroutetcp.traefik.io` or `kubectl get traefik.containo.us` for example.
+
 ### On Kubernetes with User managed Ingress
 
 To deploy a mesh ingress on Kubernetes cluster, create the AWXMeshIngress resource on the namespace where your AWX instance is running on.

docs/user-guide/database-configuration.md

@@ -56,15 +56,14 @@ If you don't have access to an external PostgreSQL service, the AWX operator can
 
 The following variables are customizable for the managed PostgreSQL service
 
-| Name                                          | Description                                   | Default                                 |
+| Name                                          | Description                                                     | Default                                 |
-| --------------------------------------------- | --------------------------------------------- | --------------------------------------- |
+| --------------------------------------------- | --------------------------------------------------------------- | --------------------------------------- |
-| postgres_image                                | Path of the image to pull                     | quay.io/sclorg/postgresql-15-c9s:latest |
+| postgres_image                                | Path of the image to pull                                       | quay.io/sclorg/postgresql-15-c9s        |
-| postgres_init_container_resource_requirements | Database init container resource requirements | requests: {cpu: 10m, memory: 64Mi}      |
+| postgres_image_version                        | Image version to pull                                           | latest                                  |
-| postgres_resource_requirements                | PostgreSQL container resource requirements    | requests: {cpu: 10m, memory: 64Mi}      |
+| postgres_resource_requirements                | PostgreSQL container (and initContainer) resource requirements  | requests: {cpu: 10m, memory: 64Mi}      |
-| postgres_storage_requirements                 | PostgreSQL container storage requirements     | requests: {storage: 8Gi}                |
+| postgres_storage_requirements                 | PostgreSQL container storage requirements                       | requests: {storage: 8Gi}                |
-| postgres_storage_class                        | PostgreSQL PV storage class                   | Empty string                            |
+| postgres_storage_class                        | PostgreSQL PV storage class                                     | Empty string                            |
-| postgres_data_path                            | PostgreSQL data path                          | `/var/lib/postgresql/data/pgdata`       |
+| postgres_priority_class                       | Priority class used for PostgreSQL pod                          | Empty string                            |
-| postgres_priority_class                       | Priority class used for PostgreSQL pod        | Empty string                            |
 
 Example of customization could be:
 
@@ -91,3 +90,30 @@ spec:
 ```
 
 **Note**: If `postgres_storage_class` is not defined, PostgreSQL will store it's data on a volume using the default storage class for your cluster.
+
+#### Note about overriding the postgres image
+
+We recommend you use the default image sclorg image. If you are coming from a deployment using the old postgres image from dockerhub (postgres:13), upgrading from awx-operator version 2.12.2 and below to 2.15.0+ will handle migrating your data to the new postgresql image (postgresql-15-c9s).
+
+You can no longer configure a custom `postgres_data_path` because it is hardcoded in the quay.io/sclorg/postgresql-15-c9s image.
+
+If you override the postgres image to use a custom postgres image like postgres:15 for example, the default data directory path may be different. These images cannot be used interchangeably.
+
+#### Initialize Postgres data volume
+
+When using a hostPath backed PVC and some other storage classes like longhorn storagfe, the postgres data directory needs to be accessible by the user in the postgres pod (UID 26).
+
+To initialize this directory with the correct permissions, configure the following setting, which will use an init container to set the permissions in the postgres volume.
+
+```yaml
+spec:
+  postgres_data_volume_init: true
+```
+
+Should you need to modify the init container commands, there is an example below.
+
+```yaml
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+```

molecule/default/tasks/awx_replicas_test.yml

@@ -49,16 +49,16 @@
 ####
 
     - debug:
-        msg: test - replicas=3 should give 3 of each
+        msg: test - replicas=2 should give 2 of each
 
     - include_tasks: apply_awx_spec.yml
       vars:
         additional_fields:
-          replicas: 3
+          replicas: 2
 
     - include_tasks: _test_case_replicas.yml
       vars:
-        expected_web_replicas: 3
+        expected_web_replicas: 2
-        expected_task_replicas: 3
+        expected_task_replicas: 2
   tags:
     - replicas

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -32,7 +32,6 @@ spec:
       memory: 16M
   no_log: false
   postgres_resource_requirements: {}
-  postgres_init_container_resource_requirements: {}
   redis_resource_requirements: {}
   additional_labels:
   - my/team

molecule/default/verify.yml

@@ -14,7 +14,8 @@
         - name: Import all test files from tasks/
           include_tasks: '{{ item }}'
           with_fileglob:
-            - tasks/*_test.yml
+            - tasks/awx_test.yml
+            - tasks/awx_replicas_test.yml
           tags:
             - always
       rescue:

noxfile.py

@@ -0,0 +1,20 @@
+import nox
+
+
+@nox.session
+def build(session: nox.Session):
+    """
+    Build the AWX Operator docsite.
+    """
+    session.install(
+        "-r",
+        "docs/requirements.in",
+        "-c",
+        "docs/requirements.txt",
+    )
+    session.run(
+        "mkdocs",
+        "build",
+        "--strict",
+        *session.posargs,
+    )

roles/installer/defaults/main.yml

@@ -296,6 +296,11 @@ replicas: 1
 web_replicas: ''
 task_replicas: ''
 
+web_liveness_period: 0
+web_readiness_period: 0
+task_liveness_period: 0
+task_readiness_period: 0
+
 task_args:
   - /usr/bin/launch_awx_task.sh
 task_command: []
@@ -395,13 +400,9 @@ postgres_resource_requirements:
   requests:
     cpu: 10m
     memory: 64Mi
-postgres_init_container_resource_requirements:
+
-  requests:
-    cpu: 10m
-    memory: 64Mi
 # Assign a preexisting priority class to the postgres pod
 postgres_priority_class: ''
-postgres_data_path: '/var/lib/postgresql/data/pgdata'
 
 # Persistence to the AWX project data folder
 # Whether or not the /var/lib/projects directory will be persistent
@@ -413,6 +414,11 @@ projects_existing_claim: ''
 # Define postgres configuration arguments to use
 postgres_extra_args: ''
 
+postgres_data_volume_init: false
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+
 # Configure postgres connection keepalive
 postgres_keepalives: true
 postgres_keepalives_idle: 5
@@ -484,3 +490,14 @@ nginx_worker_processes: 1
 nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
 nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
+
+# metrics-utility (github.com/ansible/metrics-utility)
+_metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}"
+_metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}"
+_metrics_utility_image: "{{ metrics_utility_image | default(_image) }}"
+_metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}"
+_metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}"
+_metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}"
+_metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}"
+_metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}"
+_metrics_utility_cronjob_report_schedule: "{{ metrics_utility_cronjob_report_schedule | default('@monthly') }}"

roles/installer/tasks/database_configuration.yml

@@ -129,7 +129,7 @@
       set_fact:
         sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
           sort(attribute='metadata.name') |
-          reverse }}"
+          reverse | list }}"
       when: filtered_old_postgres_pods | length
 
 
@@ -156,17 +156,17 @@
 
 - name: Check if postgres pod is running an older version
   block:
-    - name: Set path to PG_VERSION file for given container image
-      set_fact:
-        path_to_pg_version: '{{ postgres_data_path }}/PG_VERSION'
-
     - name: Get old PostgreSQL version
       k8s_exec:
         namespace: "{{ ansible_operator_meta.namespace }}"
         pod: "{{ old_postgres_pod['metadata']['name'] }}"
         command: |
           bash -c """
-          cat {{ path_to_pg_version }}
+          if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then
+            cat "{{ _postgres_data_path }}/PG_VERSION"
+          elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then
+            cat '/var/lib/postgresql/data/pgdata/PG_VERSION'
+          fi
           """
       register: _old_pg_version
 

roles/installer/tasks/enable_metrics_utility.yml

@@ -0,0 +1,23 @@
+---
+# Check to make sure provided pvc exists, error loudly if not.  Otherwise, the management pod will just stay in pending state forever.
+- name: Check provided PVC claim exists
+  kubernetes.core.k8s_info:
+    name: "{{ _metrics_utility_pvc_claim }}"
+    kind: PersistentVolumeClaim
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  when:
+    - _metrics_utility_pvc_claim | length
+
+- name: Create PVC for metrics-utility
+  kubernetes.core.k8s:
+    kind: PersistentVolumeClaim
+    definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+
+- name: Create Kubernetes CronJobs for metrics-utility
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', item) }}"
+    apply: true
+    wait: true
+  loop:
+    - cronjobs/metrics-utility-gather.yaml.j2
+    - cronjobs/metrics-utility-report.yaml.j2

roles/installer/tasks/install.yml

@@ -102,6 +102,10 @@
   include_tasks: initialize_django.yml
   when: awx_web_pod_name != ''
 
+- name: Enable optional metrics-utility
+  include_tasks: enable_metrics_utility.yml
+  when: metrics_utility_enabled | bool
+
 - name: Update status variables
   include_tasks: update_status.yml
 

roles/installer/tasks/resources_configuration.yml

@@ -266,6 +266,11 @@
         field_selectors:
           - status.phase=Running
       register: _new_pod
+      until:
+        - "_new_pod['resources'] | length"
+        - "_new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | length"
+      retries: 60
+      delay: 5
 
     - name: Update new resource pod as a variable.
       set_fact:

roles/installer/tasks/upgrade_postgres.yml

@@ -153,16 +153,3 @@
   loop:
     - "{{ ansible_operator_meta.name }}-postgres"
     - "{{ ansible_operator_meta.name }}-postgres-13"
-
-- name: Remove old persistent volume claim
-  k8s:
-    kind: PersistentVolumeClaim
-    api_version: v1
-    namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ item }}"
-    state: absent
-  loop:
-    - "postgres-{{ ansible_operator_meta.name }}-postgres-0"
-    - "postgres-{{ ansible_operator_meta.name }}-postgres-13-0"
-    - "postgres-13-{{ ansible_operator_meta.name }}-postgres-13-0"
-  when: postgres_keep_pvc_after_upgrade

roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2

@@ -0,0 +1,76 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+    {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
+    {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
+spec:
+  schedule: "{{ _metrics_utility_cronjob_gather_schedule }}"
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather'
+            {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
+            {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
+        spec:
+{% if image_pull_secret is defined %}
+          imagePullSecrets:
+            - name: {{ image_pull_secret }}
+{% elif image_pull_secrets | length > 0 %}
+          imagePullSecrets:
+{% for secret in image_pull_secrets %}
+            - name: {{ secret }}
+{% endfor %}
+{% endif %}
+          containers:
+          - name: {{ ansible_operator_meta.name }}-metrics-utility-gather
+            image: "{{ _metrics_utility_image }}"
+            imagePullPolicy: "{{ image_pull_policy }}"
+            resources:
+              requests:
+                memory: "100Mi"
+                cpu: "100m"
+            command:
+              - /bin/sh
+              - -c
+              - metrics-utility gather_automation_controller_billing_data --ship --until=10m
+            envFrom:
+            - configMapRef:
+                name: {{ _metrics_utility_configmap }}
+            volumeMounts:
+            - name: {{ ansible_operator_meta.name }}-metrics-utility
+              mountPath: /metrics-utility
+              readOnly: false
+            - name: "{{ ansible_operator_meta.name }}-application-credentials"
+              mountPath: "/etc/tower/conf.d/credentials.py"
+              subPath: credentials.py
+              readOnly: true
+            - name: {{ ansible_operator_meta.name }}-settings
+              mountPath: /etc/tower/settings.py
+              subPath: settings.py
+              readOnly: true
+          volumes:
+          - name: {{ ansible_operator_meta.name }}-metrics-utility
+            persistentVolumeClaim:
+              claimName: {{ _metrics_utility_pvc_claim }}
+              readOnly: false
+          - name: "{{ ansible_operator_meta.name }}-application-credentials"
+            secret:
+              secretName: "{{ ansible_operator_meta.name }}-app-credentials"
+              items:
+                - key: credentials.py
+                  path: 'credentials.py'
+          - name: {{ ansible_operator_meta.name }}-settings
+            configMap:
+              name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'
+              items:
+                - key: settings
+                  path: settings.py
+          restartPolicy: OnFailure

roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2

@@ -0,0 +1,76 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ ansible_operator_meta.name }}-metrics-utility-report
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+    {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
+    {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }}
+spec:
+  schedule: "{{ _metrics_utility_cronjob_report_schedule }}"
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report'
+            {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=12) | trim }}
+            {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }}
+        spec:
+{% if image_pull_secret is defined %}
+          imagePullSecrets:
+            - name: {{ image_pull_secret }}
+{% elif image_pull_secrets | length > 0 %}
+          imagePullSecrets:
+{% for secret in image_pull_secrets %}
+            - name: {{ secret }}
+{% endfor %}
+{% endif %}
+          containers:
+          - name: {{ ansible_operator_meta.name }}-metrics-utility-report
+            image: "{{ _metrics_utility_image }}"
+            imagePullPolicy: "{{ image_pull_policy }}"
+            resources:
+              requests:
+                memory: "100Mi"
+                cpu: "100m"
+            command:
+              - /bin/sh
+              - -c
+              - metrics-utility build_report
+            envFrom:
+            - configMapRef:
+                name: {{ _metrics_utility_configmap }}
+            volumeMounts:
+            - name: {{ ansible_operator_meta.name }}-metrics-utility
+              mountPath: /metrics-utility
+              readOnly: false
+            - name: "{{ ansible_operator_meta.name }}-application-credentials"
+              mountPath: "/etc/tower/conf.d/credentials.py"
+              subPath: credentials.py
+              readOnly: true
+            - name: {{ ansible_operator_meta.name }}-settings
+              mountPath: /etc/tower/settings.py
+              subPath: settings.py
+              readOnly: true
+          volumes:
+          - name: {{ ansible_operator_meta.name }}-metrics-utility
+            persistentVolumeClaim:
+              claimName: {{ _metrics_utility_pvc_claim }}
+              readOnly: false
+          - name: "{{ ansible_operator_meta.name }}-application-credentials"
+            secret:
+              secretName: "{{ ansible_operator_meta.name }}-app-credentials"
+              items:
+                - key: credentials.py
+                  path: 'credentials.py'
+          - name: {{ ansible_operator_meta.name }}-settings
+            configMap:
+              name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'
+              items:
+                - key: settings
+                  path: settings.py
+          restartPolicy: OnFailure

roles/installer/templates/deployments/web.yaml.j2

@@ -178,11 +178,10 @@ spec:
 {% endif %}
 {% if web_readiness_period|int > 0 %}
           readinessProbe:
-            exec:
+            httpGet:
-              httpGet:
+              path: /api/v2/ping/
-                path: /api/v2/ping/
+              scheme: HTTP
-                scheme: HTTP
+              port: 8052
-                port: 8052
             initialDelaySeconds: {{ web_readiness_initial_delay }}
             periodSeconds: {{ web_readiness_period }}
             failureThreshold: {{ web_readiness_failure_threshold }}

roles/installer/templates/jobs/migration.yaml.j2

@@ -32,6 +32,16 @@ spec:
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
+{% endif %}
+      serviceAccountName: '{{ ansible_operator_meta.name }}'
+{% if image_pull_secret is defined %}
+      imagePullSecrets:
+        - name: {{ image_pull_secret }}
+{% elif image_pull_secrets | length > 0 %}
+      imagePullSecrets:
+{% for secret in image_pull_secrets %}
+        - name: {{ secret }}
+{% endfor %}
 {% endif %}
       volumes:
         - name: "{{ ansible_operator_meta.name }}-application-credentials"

roles/installer/templates/statefulsets/postgres.yaml.j2

@@ -46,6 +46,27 @@ spec:
 {% endif %}
 {% if postgres_priority_class is defined %}
       priorityClassName: '{{ postgres_priority_class }}'
+{% endif %}
+{% if postgres_data_volume_init and not is_openshift %}
+      initContainers:
+        - name: init
+          image: '{{ _postgres_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          securityContext:
+            runAsUser: 0
+          command:
+            - /bin/sh
+            - -c
+            - |
+              {{ postgres_init_container_commands | indent(width=14) }}
+          resources: {{ postgres_init_container_resource_requirements | default(postgres_resource_requirements) }}
+          volumeMounts:
+            - name: postgres-{{ supported_pg_version }}
+              mountPath: '{{ _postgres_data_path | dirname }}'
+              subPath: '{{ _postgres_data_path | dirname | basename }}'
+{% if postgres_extra_volume_mounts %}
+            {{ postgres_extra_volume_mounts | indent(width=12, first=True) }}
+{% endif %}
 {% endif %}
       containers:
         - image: '{{ _postgres_image }}'
@@ -56,6 +77,7 @@ spec:
             {{ postgres_security_context_settings | to_nice_yaml | indent(12) }}
 {% endif %}
 {% if postgres_extra_args %}
+          command: ["run-postgresql"]
           args: {{ postgres_extra_args }}
 {% endif %}
           env:
@@ -93,7 +115,7 @@ spec:
                   name: '{{ __postgres_configuration_secret }}'
                   key: password
             - name: PGDATA
-              value: '{{ postgres_data_path }}'
+              value: '{{ _postgres_data_path }}'
             - name: POSTGRES_INITDB_ARGS
               value: '{{ postgres_initdb_args }}'
             - name: POSTGRES_HOST_AUTH_METHOD
@@ -110,9 +132,9 @@ spec:
               name: postgres-{{ supported_pg_version }}
           volumeMounts:
             - name: postgres-{{ supported_pg_version }}
-              mountPath: '{{ postgres_data_path | dirname }}'
+              mountPath: '{{ _postgres_data_path | dirname }}'
-              subPath: '{{ postgres_data_path | dirname | basename }}'
+              subPath: '{{ _postgres_data_path | dirname | basename }}'
-{% if postgres_extra_volume_mounts -%}
+{% if postgres_extra_volume_mounts %}
             {{ postgres_extra_volume_mounts | indent(width=12, first=True) }}
 {% endif %}
           resources: {{ postgres_resource_requirements }}

roles/installer/templates/storage/metrics-utility.yaml.j2

@@ -0,0 +1,18 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ _metrics_utility_pvc_claim }}
+  namespace: {{ ansible_operator_meta.namespace }}
+  ownerReferences: null
+  labels:
+    {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ _metrics_utility_pvc_claim_size }}
+{% if metrics_utility_pvc_claim_storage_class is defined %}
+  storageClassName: {{ metrics_utility_pvc_claim_storage_class }}
+{% endif %}

roles/installer/vars/main.yml

@@ -7,3 +7,4 @@ projects_existing_claim: ''
 supported_pg_version: 15
 _previous_upgraded_pg_version: 0
 old_postgres_pod: []
+_postgres_data_path: '/var/lib/pgsql/data/userdata'

roles/mesh_ingress/templates/ingress.yml.j2

@@ -70,7 +70,7 @@ metadata:
   name: {{ ansible_operator_meta.name }}
   namespace: {{ ansible_operator_meta.namespace }}
 spec:
-{% if external_hostname %}
+{% if external_hostname is defined %}
   host: {{ external_hostname }}
 {% endif %}
   port: