Set default for -iness probe parameters and add docs (#1808)

Add postgres init container if
postgres_data_volume_init is true

This is aimed to solve the issue where users may
need to chmod or chown the postgres
data volume for user 26, which is the user
that is running postgres in the sclorg image.

For example, one can now set the follow on the AWX spec:
spec:
  postgres_init_container_commands: |
    chown 26:0 /var/lib/pgsql/data
    chmod 700 /var/lib/pgsql/data

Deprecate postgres_init_container_resource_requirements param in favor
of postgres_resource_requirements param.

Signed-off-by: Seth Foster <fosterbseth@gmail.com>
Co-authored-by: craph <14820052+craph@users.noreply.github.com>
Co-authored-by: kurokobo <kuro664@gmail.com>
Co-authored-by: Christian M. Adams <chadams@redhat.com>

.github/workflows/ci.yaml

@@ -99,3 +99,5 @@ jobs:
             echo "${no_log}"
             exit 1
           fi
+  nox-sessions:
+    uses: ./.github/workflows/reusable-nox.yml

.github/workflows/reusable-nox.yml

@@ -0,0 +1,26 @@
+---
+name: nox
+
+"on":
+    workflow_call:
+
+jobs:
+    nox:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                - session: build
+                  python-versions: "3.11"
+        name: "Run nox ${{ matrix.session }} session"
+        steps:
+        - name: Check out repo
+          uses: actions/checkout@v4
+        - name: Setup nox
+          uses: wntrblm/nox@2024.03.02
+          with:
+              python-versions: "${{ matrix.python-versions }}"
+        - name: "Run nox -s ${{ matrix.session }}"
+          run: |
+            nox -s "${{ matrix.session }}"

.helm/starter/templates/storage/postgres-pv.yaml

@@ -13,7 +13,7 @@ spec:
     storage: {{ default "8Gi" .size | quote }}
   storageClassName: {{ include "postgres.storageClassName" $ }}
   hostPath:
-    path: {{ required "customVolumes.postgres.hostPath or spec.postgres_data_path are required!" (default ($.Values.AWX.spec).postgres_data_path .hostPath) | quote }}
+    path: /var/lib/pgsql/data/userdata
 {{- end }}
 {{- end }}
 {{- end }}

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1460,7 +1460,7 @@ spec:
                     type: object
                 type: object
               postgres_init_container_resource_requirements:
-                description: Resource requirements for the postgres init container
+                description: (Deprecated, use postgres_resource_requirements parameter) Resource requirements for the postgres init container
                 properties:
                   requests:
                     properties:
@@ -1807,13 +1807,16 @@ spec:
               postgres_priority_class:
                 description: Assign a preexisting priority class to the postgres pod
                 type: string
-              postgres_data_path:
-                description: Path where the PostgreSQL data are located
-                type: string
               postgres_extra_args:
                 type: array
                 items:
                   type: string
+              postgres_data_volume_init:
+                description: Sets permissions on the /var/lib/pgdata/data for postgres container using an init container (not Openshift)
+                type: boolean
+              postgres_init_container_commands:
+                description: Customize the postgres init container commands (Non Openshift)
+                type: string
               postgres_extra_volumes:
                 description: Specify extra volumes to add to the application pod
                 type: string

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -456,12 +456,21 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
-      - description: The PostgreSQL init container is not used when an external DB
+      - description: Sets permissions on the /var/lib/pgsql/data for postgres container using an init container (not Openshift)
-          is configured
+        displayName: PostgreSQL initialize data volume
+        path: postgres_data_volume_init
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: Customize the postgres init container commands (Non Openshift)
+        displayName: PostgreSQL Init Container Commands
+        path: postgres_init_container_commands
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: (Deprecated, use postgres_resource_requirements parameter instead)
         displayName: PostgreSQL Init Container Resource Requirements
         path: postgres_init_container_resource_requirements
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
       - displayName: Redis Container Resource Requirements
         path: redis_resource_requirements
@@ -637,11 +646,6 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:advanced
-      - displayName: Postgres Datapath
-        path: postgres_data_path
-        x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
-        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Postgres Extra Arguments
         path: postgres_extra_args
         x-descriptors:
@@ -1034,6 +1038,7 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
       - displayName: Metrics-Utlity Image Version
         path: metrics_utility_image_version
         x-descriptors:

config/samples/awx_v1beta1_awx_resource_limits.yaml

@@ -46,10 +46,3 @@ spec:
     limits:
       cpu: 1000m
       memory: 2Gi
-  postgres_init_container_resource_requirements:
-    requests:
-      cpu: 10m
-      memory: 64Mi
-    limits:
-      cpu: 1000m
-      memory: 2Gi

docs/user-guide/advanced-configuration/container-probes.md

@@ -2,6 +2,17 @@
 These parameters control the usage of liveness and readiness container probes for
 the web and task containers.
 
+> [!ALERT]
+> All of probes are disabled by default for now, to enable it, set the *_period parameters.  For example:
+
+```
+
+web_liveness_period: 15
+web_readiness_period: 15
+task_liveness_period: 15
+task_readiness_period: 15
+```
+
 #### Web / Task Container Liveness Check
 
 The liveness probe queries the status of the supervisor daemon of the container.  The probe will fail if it

docs/user-guide/advanced-configuration/containers-resource-requirements.md

@@ -22,16 +22,15 @@ spec:
 
 The resource requirements for both, the task and the web containers are configurable - both the lower end (requests) and the upper end (limits).
 
-| Name                       | Description                                      | Default                              |
+| Name                                 | Description                                                  | Default                              |
-| -------------------------- | ------------------------------------------------ | ------------------------------------ |
+| ------------------------------------ | ------------------------------------------------------------ | ------------------------------------ |
-| web_resource_requirements  | Web container resource requirements              | requests: {cpu: 100m, memory: 128Mi} |
+| web_resource_requirements            | Web container resource requirements                          | requests: {cpu: 100m, memory: 128Mi} |
-| task_resource_requirements | Task container resource requirements             | requests: {cpu: 100m, memory: 128Mi} |
+| task_resource_requirements           | Task container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
-| ee_resource_requirements   | EE control plane container resource requirements | requests: {cpu: 50m, memory: 64Mi} |
+| ee_resource_requirements             | EE control plane container resource requirements             | requests: {cpu: 50m, memory: 64Mi} |
-| redis_resource_requirements   | Redis container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
+| redis_resource_requirements          | Redis container resource requirements                        | requests: {cpu: 100m, memory: 128Mi} |
-| postgres_resource_requirements   | Postgres container resource requirements | requests: {cpu: 10m, memory: 64Mi} |
+| postgres_resource_requirements       | Postgres container (and initContainer) resource requirements | requests: {cpu: 10m, memory: 64Mi}   |
-| rsyslog_resource_requirements   | Rsyslog container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
+| rsyslog_resource_requirements        | Rsyslog container resource requirements                      | requests: {cpu: 100m, memory: 128Mi} |
-| init_container_resource_requirements   | Init Container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
+| init_container_resource_requirements | Init Container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
-| postgres_init_container_resource_requirements   | Postgres Init Container resource requirements | requests: {cpu: 10m, memory: 64Mi} |
 
 
 Example of customization could be:
@@ -85,13 +84,6 @@ spec:
     limits:
       cpu: 1000m
       memory: 2Gi
-  postgres_init_container_resource_requirements:
-    requests:
-      cpu: 10m
-      memory: 64Mi
-    limits:
-      cpu: 1000m
-      memory: 2Gi
 ```
 
 

docs/user-guide/advanced-configuration/deploying-a-specific-version-of-awx.md

@@ -2,15 +2,19 @@
 
 There are a few variables that are customizable for awx the image management.
 
-| Name                | Description               | Default                                    |
+| Name                          | Description               | Default                                    |
-| ------------------- | ------------------------- | -----------------------------------------  |
+| ----------------------------- | ------------------------- | ------------------------------------------ |
-| image               | Path of the image to pull | quay.io/ansible/awx                        |
+| image                         | Path of the image to pull | quay.io/ansible/awx                        |
-| image_version       | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| image_version                 | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
-| image_pull_policy   | The pull policy to adopt  | IfNotPresent                               |
+| image_pull_policy             | The pull policy to adopt  | IfNotPresent                               |
-| image_pull_secrets  | The pull secrets to use   | None                                       |
+| image_pull_secrets            | The pull secrets to use   | None                                       |
-| ee_images           | A list of EEs to register | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| ee_images                     | A list of EEs to register | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
-| redis_image         | Path of the image to pull | docker.io/redis                            |
+| redis_image                   | Path of the image to pull | docker.io/redis                            |
-| redis_image_version | Image version to pull     | latest                                     |
+| redis_image_version           | Image version to pull     | latest                                     |
+| control_plane_ee_image        | Image version to pull     | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| init_container_image          | Path of the image to pull | quay.io/ansible/awx-ee                     |
+| init_container_image_version  | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| init_projects_container_image | Image version to pull     | quay.io/centos/centos:stream9              |
 
 Example of customization could be:
 
@@ -26,6 +30,10 @@ spec:
   ee_images:
     - name: my-custom-awx-ee
       image: myorg/my-custom-awx-ee
+  control_plane_ee_image: myorg/my-custom-awx-ee:latest
+  init_container_image: myorg/my-custom-awx-ee
+  init_container_image_version: latest
+  init_projects_container_image: myorg/my-mirrored-centos:stream9
 ```
 
 **Note**: The `image` and `image_version` are intended for local mirroring scenarios. Please note that using a version of AWX other than the one bundled with the `awx-operator` is **not** supported. For the default values, check the [main.yml](https://github.com/ansible/awx-operator/blob/devel/roles/installer/defaults/main.yml) file.

docs/user-guide/database-configuration.md

@@ -56,15 +56,14 @@ If you don't have access to an external PostgreSQL service, the AWX operator can
 
 The following variables are customizable for the managed PostgreSQL service
 
-| Name                                          | Description                                   | Default                                 |
+| Name                                          | Description                                                     | Default                                 |
-| --------------------------------------------- | --------------------------------------------- | --------------------------------------- |
+| --------------------------------------------- | --------------------------------------------------------------- | --------------------------------------- |
-| postgres_image                                | Path of the image to pull                     | quay.io/sclorg/postgresql-15-c9s:latest |
+| postgres_image                                | Path of the image to pull                                       | quay.io/sclorg/postgresql-15-c9s        |
-| postgres_init_container_resource_requirements | Database init container resource requirements | requests: {cpu: 10m, memory: 64Mi}      |
+| postgres_image_version                        | Image version to pull                                           | latest                                  |
-| postgres_resource_requirements                | PostgreSQL container resource requirements    | requests: {cpu: 10m, memory: 64Mi}      |
+| postgres_resource_requirements                | PostgreSQL container (and initContainer) resource requirements  | requests: {cpu: 10m, memory: 64Mi}      |
-| postgres_storage_requirements                 | PostgreSQL container storage requirements     | requests: {storage: 8Gi}                |
+| postgres_storage_requirements                 | PostgreSQL container storage requirements                       | requests: {storage: 8Gi}                |
-| postgres_storage_class                        | PostgreSQL PV storage class                   | Empty string                            |
+| postgres_storage_class                        | PostgreSQL PV storage class                                     | Empty string                            |
-| postgres_data_path                            | PostgreSQL data path                          | `/var/lib/postgresql/data/pgdata`       |
+| postgres_priority_class                       | Priority class used for PostgreSQL pod                          | Empty string                            |
-| postgres_priority_class                       | Priority class used for PostgreSQL pod        | Empty string                            |
 
 Example of customization could be:
 
@@ -91,3 +90,30 @@ spec:
 ```
 
 **Note**: If `postgres_storage_class` is not defined, PostgreSQL will store it's data on a volume using the default storage class for your cluster.
+
+#### Note about overriding the postgres image
+
+We recommend you use the default image sclorg image. If you are coming from a deployment using the old postgres image from dockerhub (postgres:13), upgrading from awx-operator version 2.12.2 and below to 2.15.0+ will handle migrating your data to the new postgresql image (postgresql-15-c9s).
+
+You can no longer configure a custom `postgres_data_path` because it is hardcoded in the quay.io/sclorg/postgresql-15-c9s image.
+
+If you override the postgres image to use a custom postgres image like postgres:15 for example, the default data directory path may be different. These images cannot be used interchangeably.
+
+#### Initialize Postgres data volume
+
+When using a hostPath backed PVC and some other storage classes like longhorn storagfe, the postgres data directory needs to be accessible by the user in the postgres pod (UID 26).
+
+To initialize this directory with the correct permissions, configure the following setting, which will use an init container to set the permissions in the postgres volume.
+
+```yaml
+spec:
+  postgres_data_volume_init: true
+```
+
+Should you need to modify the init container commands, there is an example below.
+
+```yaml
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+```

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -32,7 +32,6 @@ spec:
       memory: 16M
   no_log: false
   postgres_resource_requirements: {}
-  postgres_init_container_resource_requirements: {}
   redis_resource_requirements: {}
   additional_labels:
   - my/team

noxfile.py

@@ -0,0 +1,20 @@
+import nox
+
+
+@nox.session
+def build(session: nox.Session):
+    """
+    Build the AWX Operator docsite.
+    """
+    session.install(
+        "-r",
+        "docs/requirements.in",
+        "-c",
+        "docs/requirements.txt",
+    )
+    session.run(
+        "mkdocs",
+        "build",
+        "--strict",
+        *session.posargs,
+    )

roles/installer/defaults/main.yml

@@ -296,6 +296,11 @@ replicas: 1
 web_replicas: ''
 task_replicas: ''
 
+web_liveness_period: 0
+web_readiness_period: 0
+task_liveness_period: 0
+task_readiness_period: 0
+
 task_args:
   - /usr/bin/launch_awx_task.sh
 task_command: []
@@ -395,13 +400,9 @@ postgres_resource_requirements:
   requests:
     cpu: 10m
     memory: 64Mi
-postgres_init_container_resource_requirements:
+
-  requests:
-    cpu: 10m
-    memory: 64Mi
 # Assign a preexisting priority class to the postgres pod
 postgres_priority_class: ''
-postgres_data_path: '/var/lib/pgsql/data/pgdata'
 
 # Persistence to the AWX project data folder
 # Whether or not the /var/lib/projects directory will be persistent
@@ -413,6 +414,11 @@ projects_existing_claim: ''
 # Define postgres configuration arguments to use
 postgres_extra_args: ''
 
+postgres_data_volume_init: false
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+
 # Configure postgres connection keepalive
 postgres_keepalives: true
 postgres_keepalives_idle: 5

roles/installer/tasks/database_configuration.yml

@@ -129,7 +129,7 @@
       set_fact:
         sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
           sort(attribute='metadata.name') |
-          reverse }}"
+          reverse | list }}"
       when: filtered_old_postgres_pods | length
 
 
@@ -162,8 +162,8 @@
         pod: "{{ old_postgres_pod['metadata']['name'] }}"
         command: |
           bash -c """
-          if [ -f "{{ postgres_data_path }}/PG_VERSION" ]; then
+          if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then
-            cat "{{ postgres_data_path }}/PG_VERSION"
+            cat "{{ _postgres_data_path }}/PG_VERSION"
           elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then
             cat '/var/lib/postgresql/data/pgdata/PG_VERSION'
           fi

roles/installer/tasks/resources_configuration.yml

@@ -266,6 +266,11 @@
         field_selectors:
           - status.phase=Running
       register: _new_pod
+      until:
+        - "_new_pod['resources'] | length"
+        - "_new_pod['resources'] | rejectattr('metadata.deletionTimestamp', 'defined') | length"
+      retries: 60
+      delay: 5
 
     - name: Update new resource pod as a variable.
       set_fact:

roles/installer/templates/deployments/web.yaml.j2

@@ -178,11 +178,10 @@ spec:
 {% endif %}
 {% if web_readiness_period|int > 0 %}
           readinessProbe:
-            exec:
+            httpGet:
-              httpGet:
+              path: /api/v2/ping/
-                path: /api/v2/ping/
+              scheme: HTTP
-                scheme: HTTP
+              port: 8052
-                port: 8052
             initialDelaySeconds: {{ web_readiness_initial_delay }}
             periodSeconds: {{ web_readiness_period }}
             failureThreshold: {{ web_readiness_failure_threshold }}

roles/installer/templates/statefulsets/postgres.yaml.j2

@@ -46,6 +46,27 @@ spec:
 {% endif %}
 {% if postgres_priority_class is defined %}
       priorityClassName: '{{ postgres_priority_class }}'
+{% endif %}
+{% if postgres_data_volume_init and not is_openshift %}
+      initContainers:
+        - name: init
+          image: '{{ _postgres_image }}'
+          imagePullPolicy: '{{ image_pull_policy }}'
+          securityContext:
+            runAsUser: 0
+          command:
+            - /bin/sh
+            - -c
+            - |
+              {{ postgres_init_container_commands | indent(width=14) }}
+          resources: {{ postgres_init_container_resource_requirements | default(postgres_resource_requirements) }}
+          volumeMounts:
+            - name: postgres-{{ supported_pg_version }}
+              mountPath: '{{ _postgres_data_path | dirname }}'
+              subPath: '{{ _postgres_data_path | dirname | basename }}'
+{% if postgres_extra_volume_mounts %}
+            {{ postgres_extra_volume_mounts | indent(width=12, first=True) }}
+{% endif %}
 {% endif %}
       containers:
         - image: '{{ _postgres_image }}'
@@ -94,7 +115,7 @@ spec:
                   name: '{{ __postgres_configuration_secret }}'
                   key: password
             - name: PGDATA
-              value: '{{ postgres_data_path }}'
+              value: '{{ _postgres_data_path }}'
             - name: POSTGRES_INITDB_ARGS
               value: '{{ postgres_initdb_args }}'
             - name: POSTGRES_HOST_AUTH_METHOD
@@ -111,9 +132,9 @@ spec:
               name: postgres-{{ supported_pg_version }}
           volumeMounts:
             - name: postgres-{{ supported_pg_version }}
-              mountPath: '{{ postgres_data_path | dirname }}'
+              mountPath: '{{ _postgres_data_path | dirname }}'
-              subPath: '{{ postgres_data_path | dirname | basename }}'
+              subPath: '{{ _postgres_data_path | dirname | basename }}'
-{% if postgres_extra_volume_mounts -%}
+{% if postgres_extra_volume_mounts %}
             {{ postgres_extra_volume_mounts | indent(width=12, first=True) }}
 {% endif %}
           resources: {{ postgres_resource_requirements }}

roles/installer/vars/main.yml

@@ -7,3 +7,4 @@ projects_existing_claim: ''
 supported_pg_version: 15
 _previous_upgraded_pg_version: 0
 old_postgres_pod: []
+_postgres_data_path: '/var/lib/pgsql/data/userdata'

roles/mesh_ingress/templates/ingress.yml.j2

@@ -70,7 +70,7 @@ metadata:
   name: {{ ansible_operator_meta.name }}
   namespace: {{ ansible_operator_meta.namespace }}
 spec:
-{% if external_hostname %}
+{% if external_hostname is defined %}
   host: {{ external_hostname }}
 {% endif %}
   port: