Adding keepalive while doing pg_dump (#1580)

- Added docs for securityContext 
- enabled web securityContext configuration

Co-authored-by: Christian M. Adams <chadams@redhat.com>

Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/operator-framework/ansible-operator:v1.28.1
+FROM quay.io/operator-framework/ansible-operator:v1.31.0
 
 USER 0
 

README.md

@@ -76,6 +76,7 @@ All of our usage and configuration docs are nested in the `docs/` directory. Bel
     - [Redis Container Capabilities](./docs/user-guide/advanced-configuration/redis-container-capabilities.md)
     - [Trusting a Custom Certificate Authority](./docs/user-guide/advanced-configuration/trusting-a-custom-certificate-authority.md)
     - [Service Account](./docs/user-guide/advanced-configuration/service-account.md)
+    - [Security Context](./docs/user-guide/advanced-configuration/security-context.md)
     - [Persisting the Projects Directory](./docs/user-guide/advanced-configuration/persisting-projects-directory.md)
 - Troubleshooting
   - [General Debugging](./docs/troubleshooting/debugging.md)
@@ -108,5 +109,6 @@ We ask all of our community members and contributors to adhere to the [Ansible c
 
 We welcome your feedback and ideas. The AWX operator uses the same mailing list and IRC channel as AWX itself. Here's how to reach us with feedback and questions:
 
-- Join the `#ansible-awx` channel on irc.libera.chat
+- Join the [Ansible AWX channel on Matrix](https://matrix.to/#/#awx:ansible.com)
+- Join the [Ansible Community Forum](https://forum.ansible.com)
 - Join the [mailing list](https://groups.google.com/forum/#!forum/awx-project)

config/crd/bases/awx.ansible.com_awxbackups.yaml

@@ -90,6 +90,10 @@ spec:
               postgres_image_version:
                 description: PostgreSQL container image version to use
                 type: string
+              precreate_partition_hours:
+                description: Number of hours worth of events table partitions to precreate before backup to avoid pg_dump locks.
+                type: integer
+                format: int32
               image_pull_policy:
                 description: The image pull policy
                 type: string

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -63,21 +63,29 @@ spec:
               admin_password_secret:
                 description: Secret where the admin password can be found
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               postgres_configuration_secret:
                 description: Secret where the database configuration can be found
                 type: string
               old_postgres_configuration_secret:
                 description: Secret where the old database configuration can be found for data migration
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               postgres_label_selector:
                 description: Label selector used to identify postgres pod for data migration
                 type: string
               secret_key_secret:
                 description: Secret where the secret key can be found
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               broadcast_websocket_secret:
                 description: Secret where the broadcast websocket secret can be found
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               extra_volumes:
                 description: Specify extra volumes to add to the application pod
                 type: string
@@ -1771,6 +1779,10 @@ spec:
               session_cookie_secure:
                 description: Set session cookie secure mode for web
                 type: string
+              postgres_security_context_settings:
+                description: Key/values that will be set under the pod-level securityContext field
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
               receptor_log_level:
                 description: Set log level of receptor service
                 type: string

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -50,11 +50,22 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Precreate Partition Hours
+        path: precreate_partition_hours
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Database Backup Label Selector
         path: postgres_label_selector
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: PostgreSQL Security Context Settings
+        path: postgres_security_context_settings
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden      
       - displayName: PostgreSQL Image
         path: postgres_image
         x-descriptors:

docs/contributors-guide/release-process.md

@@ -8,3 +8,18 @@ After the draft release is created, publish it and the [Promote AWX Operator ima
 
 - Publish image to Quay
 - Release Helm chart
+
+After the GHA is complete, the final step is to run the [publish-to-operator-hub.sh](./hack/publish-to-operator-hub.sh) script, which will create a PR in the following repos to add the new awx-operator bundle version to OperatorHub:
+* https://github.com/k8s-operatorhub/community-operators (community operator index)
+* https://github.com/redhat-openshift-ecosystem/community-operators-prod (operator index shipped with Openshift)
+
+The usage is documented in the script itself, but here is an example of how you would use the script to publish the 2.5.3 awx-opeator bundle to OperatorHub.
+Note that you need to specify the version being released, as well as the previous version. This is because the bundle has a pointer to the previous version that is it being upgrade from. This is used by OLM to create a dependency graph.
+
+```bash
+$ VERSION=2.5.3 PREV_VERSION=2.5.2 ./publish-operator.sh
+```
+
+> Note: There are some quirks with running this on OS X that still need to be fixed, but the script runs smoothly on linux.
+
+As soon as CI completes successfully, the PR's will be auto-merged. Please remember to monitor those PR's to make sure that CI passes, sometimes it needs a retry.

docs/user-guide/advanced-configuration/security-context.md

@@ -0,0 +1,27 @@
+#### Service Account
+
+It is possible to modify some `SecurityContext` proprieties of the various deployments and stateful sets if needed.
+
+| Name                               | Description                                  | Default |
+| ---------------------------------- | -------------------------------------------- | ------- |
+| security_context_settings          | SecurityContext for Task and Web deployments | {}      |
+| postgres_security_context_settings | SecurityContext for Task and Web deployments | {}      |
+
+
+Example configuration securityContext for the Task and Web deployments:
+
+```yaml
+spec:
+  security_context_settings:
+    allowPrivilegeEscalation: false
+    capabilities:
+        drop:
+        - ALL
+```
+
+
+```yaml
+spec:
+  postgres_security_context_settings:
+    runAsNonRoot: true
+```

requirements.yml

@@ -3,4 +3,4 @@ collections:
   - name: kubernetes.core
     version: '>=2.3.2'
   - name: operator_sdk.util
-    version: "0.4.0"
+    version: "0.5.0"

roles/backup/defaults/main.yml

@@ -44,4 +44,7 @@ additional_labels: []
 
 # Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
 set_self_labels: true
+
+# Number of whole hours worth of events table partitions to precreate before starting backup to avoid pg_dump locks.
+precreate_partition_hours: 3
 ...

roles/backup/tasks/postgres.yml

@@ -82,6 +82,41 @@
     resolvable_db_host: '{{ (awx_postgres_type == "managed") | ternary(awx_postgres_host + "." + ansible_operator_meta.namespace + ".svc.cluster.local", awx_postgres_host) }}'  # yamllint disable-line rule:line-length
   no_log: "{{ no_log }}"
 
+- name: Get the current resource task pod information.
+  k8s_info:
+    api_version: v1
+    kind: Pod
+    namespace: '{{ ansible_operator_meta.namespace }}'
+    label_selectors:
+      - "app.kubernetes.io/name={{ deployment_name }}-task"
+      - "app.kubernetes.io/managed-by={{ deployment_type }}-operator"
+      - "app.kubernetes.io/component={{ deployment_type }}"
+    field_selectors:
+      - status.phase=Running
+  register: awx_task_pod
+
+- name: Set the resource pod as a variable.
+  set_fact:
+    awx_task_pod: >-
+      {{ awx_task_pod['resources']
+         | rejectattr('metadata.deletionTimestamp', 'defined')
+         | sort(attribute='metadata.creationTimestamp')
+         | first | default({}) }}
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    awx_task_pod_name: "{{ awx_task_pod['metadata']['name'] | default('') }}"
+
+- name: Precreate database partitions
+  k8s_exec:
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    pod: "{{ awx_task_pod_name }}"
+    container: "{{ deployment_name }}-task"
+    command: awx-manage precreate_partitions --count='{{ precreate_partition_hours }}'
+  when: precreate_partition_hours > 0
+  register: result
+  changed_when: "'Created partitions for' in result.stdout"
+
 - name: Set pg_dump command
   set_fact:
     pgdump: >-
@@ -99,11 +134,27 @@
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Dumping data from database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
       PGPASSWORD='{{ awx_postgres_pass }}' {{ pgdump }} > {{ backup_dir }}/tower.db
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
   failed_when: "'Successful' not in data_migration.stdout"

roles/installer/defaults/main.yml

@@ -424,6 +424,7 @@ garbage_collect_secrets: false
 development_mode: false
 
 security_context_settings: {}
+postgres_security_context_settings: {}
 
 # Set no_log settings on certain tasks
 no_log: true

roles/installer/templates/deployments/task.yaml.j2

@@ -442,7 +442,7 @@ spec:
         fsGroup: 1000
 {% endif %}
 {% if security_context_settings|length %}
-        {{ security_context_settings | to_nice_yaml | indent(8) }}
+        {{ security_context_settings | to_nice_yaml | indent(10) }}
 {% endif %}
 {% endif %}
 {% if termination_grace_period_seconds is defined %}

roles/installer/templates/deployments/web.yaml.j2

@@ -340,6 +340,10 @@ spec:
 {% elif affinity %}
       affinity:
         {{ affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
+{% if security_context_settings|length %}
+      securityContext:
+        {{ security_context_settings | to_nice_yaml | indent(8) }}
 {% endif %}
       volumes:
         - name: "{{ ansible_operator_meta.name }}-receptor-ca"

roles/installer/templates/statefulsets/postgres.yaml.j2

@@ -51,6 +51,10 @@ spec:
         - image: '{{ _postgres_image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
           name: postgres
+{% if postgres_security_context_settings|length %}
+          securityContext:
+            {{ postgres_security_context_settings | to_nice_yaml | indent(12) }}
+{% endif %}
 {% if postgres_extra_args %}
           args: {{ postgres_extra_args }}
 {% endif %}