Pre-create event table partitions before db backup in operator (#1443)

Signed-off-by: jessicamack <jmack@redhat.com>
<44379968+TheRealHaoLiu@users.noreply.github.com>

.github/workflows/ci.yaml

@@ -18,9 +18,9 @@ jobs:
     env:
       DOCKER_API_VERSION: "1.41"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -45,12 +45,12 @@ jobs:
     runs-on: ubuntu-latest
     name: helm
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.8.0
 
       - name: Build operator image and load into kind
         run: |
@@ -88,7 +88,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Check no_log statements
         run: |

.github/workflows/devel.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Push devel image
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Build Image
         run: |

.github/workflows/label_issue.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Label Issue - Community
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
       - name: Install python requests
         run: pip install requests

.github/workflows/label_pr.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Label PR - Community
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
       - name: Install python requests
         run: pip install requests

.github/workflows/stage.yml

@@ -38,13 +38,13 @@ jobs:
           exit 0
 
       - name: Checkout awx
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ github.repository_owner }}/awx
           path: awx
 
       - name: Checkout awx-operator
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ github.repository_owner }}/awx-operator
           path: awx-operator

README.md

@@ -47,7 +47,7 @@ All of our usage and configuration docs are nested in the `docs/` directory. Bel
   - [Authors](./docs/contributors-guide/author.md)
 - Installation
   - [Basic Install](./docs/installation/basic-install.md)
-  - [Creating a Minikube cluster for testing](./docs/creating-a-minikube-cluster-for-testing.md)
+  - [Creating a Minikube cluster for testing](./docs/installation/creating-a-minikube-cluster-for-testing.md)
   - [Helm Install](./docs/installation/helm-install-on-existing-cluster.md)
 - [Migration](./docs/migration/migration.md)
 - [Uninstall](./docs/uninstall/uninstall.md)
@@ -56,7 +56,7 @@ All of our usage and configuration docs are nested in the `docs/` directory. Bel
   - [Database Configuration](./docs/user-guide/database-configuration.md)
   - [Network and TLS Configuration](./docs/user-guide/network-and-tls-configuration.md)
   - Advanced Configuration
-    - [No Log](./docs/no-log.md)
+    - [No Log](./docs/user-guide/advanced-configuration/no-log.md)
     - [Deploy a Specific Version of AWX](./docs/user-guide/advanced-configuration/deploying-a-specific-version-of-awx.md)
     - [Resource Requirements](./docs/user-guide/advanced-configuration/containers-resource-requirements.md)
     - [Extra Settings](./docs/user-guide/advanced-configuration/extra-settings.md)
@@ -108,5 +108,6 @@ We ask all of our community members and contributors to adhere to the [Ansible c
 
 We welcome your feedback and ideas. The AWX operator uses the same mailing list and IRC channel as AWX itself. Here's how to reach us with feedback and questions:
 
-- Join the `#ansible-awx` channel on irc.libera.chat
+- Join the [Ansible AWX channel on Matrix](https://matrix.to/#/#awx:ansible.com)
+- Join the [Ansible Community Forum](https://forum.ansible.com)
 - Join the [mailing list](https://groups.google.com/forum/#!forum/awx-project)

config/crd/bases/awx.ansible.com_awxbackups.yaml

@@ -90,6 +90,9 @@ spec:
               postgres_image_version:
                 description: PostgreSQL container image version to use
                 type: string
+              precreate_partition_hours:
+                description: Number of hours worth of events table partitions to precreate before backup to avoid pg_dump locks.
+                type: string
               image_pull_policy:
                 description: The image pull policy
                 type: string

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -63,21 +63,29 @@ spec:
               admin_password_secret:
                 description: Secret where the admin password can be found
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               postgres_configuration_secret:
                 description: Secret where the database configuration can be found
                 type: string
               old_postgres_configuration_secret:
                 description: Secret where the old database configuration can be found for data migration
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               postgres_label_selector:
                 description: Label selector used to identify postgres pod for data migration
                 type: string
               secret_key_secret:
                 description: Secret where the secret key can be found
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               broadcast_websocket_secret:
                 description: Secret where the broadcast websocket secret can be found
                 type: string
+                maxLength: 255
+                pattern: '^[a-zA-Z0-9][-a-zA-Z0-9]{0,253}[a-zA-Z0-9]$'
               extra_volumes:
                 description: Specify extra volumes to add to the application pod
                 type: string

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -50,6 +50,12 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Precreate Partition Hours
+        path: precreate_partition_hours
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Database Backup Label Selector
         path: postgres_label_selector
         x-descriptors:

docs/contributors-guide/release-process.md

@@ -8,3 +8,18 @@ After the draft release is created, publish it and the [Promote AWX Operator ima
 
 - Publish image to Quay
 - Release Helm chart
+
+After the GHA is complete, the final step is to run the [publish-to-operator-hub.sh](./hack/publish-to-operator-hub.sh) script, which will create a PR in the following repos to add the new awx-operator bundle version to OperatorHub:
+* https://github.com/k8s-operatorhub/community-operators (community operator index)
+* https://github.com/redhat-openshift-ecosystem/community-operators-prod (operator index shipped with Openshift)
+
+The usage is documented in the script itself, but here is an example of how you would use the script to publish the 2.5.3 awx-opeator bundle to OperatorHub.
+Note that you need to specify the version being released, as well as the previous version. This is because the bundle has a pointer to the previous version that is it being upgrade from. This is used by OLM to create a dependency graph.
+
+```bash
+$ VERSION=2.5.3 PREV_VERSION=2.5.2 ./publish-operator.sh
+```
+
+> Note: There are some quirks with running this on OS X that still need to be fixed, but the script runs smoothly on linux.
+
+As soon as CI completes successfully, the PR's will be auto-merged. Please remember to monitor those PR's to make sure that CI passes, sometimes it needs a retry.

molecule/default/tasks/awx_replicas_test.yml

@@ -1,64 +1,64 @@
 ---
 - block:
-  - debug:
-      msg: test - web_replicas and task_replicas should override replicas
+    - debug:
+        msg: test - web_replicas and task_replicas should override replicas
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 2
-        web_replicas: 0
-        task_replicas: 0
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 2
+          web_replicas: 0
+          task_replicas: 0
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 0
-      expected_task_replicas: 0
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 0
+        expected_task_replicas: 0
 
 ####
 
-  - debug:
-      msg: test - replicas should act as a default
+    - debug:
+        msg: test - replicas should act as a default
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 2
-        web_replicas: 1
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 2
+          web_replicas: 1
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 1
-      expected_task_replicas: 2
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 1
+        expected_task_replicas: 2
 
 ####
 
-  - debug:
-      msg: test - replicas=0 should kill all pods
+    - debug:
+        msg: test - replicas=0 should kill all pods
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 0
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 0
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 0
-      expected_task_replicas: 0
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 0
+        expected_task_replicas: 0
 
 ####
 
-  - debug:
-      msg: test - replicas=3 should give 3 of each
+    - debug:
+        msg: test - replicas=3 should give 3 of each
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 3
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 3
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 3
-      expected_task_replicas: 3
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 3
+        expected_task_replicas: 3
   tags:
     - replicas

roles/backup/defaults/main.yml

@@ -44,4 +44,7 @@ additional_labels: []
 
 # Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
 set_self_labels: true
+
+# Number of whole hours worth of events table partitions to precreate before starting backup to avoid pg_dump locks.
+precreate_partition_hours: 3
 ...

roles/backup/tasks/postgres.yml

@@ -82,6 +82,41 @@
     resolvable_db_host: '{{ (awx_postgres_type == "managed") | ternary(awx_postgres_host + "." + ansible_operator_meta.namespace + ".svc.cluster.local", awx_postgres_host) }}'  # yamllint disable-line rule:line-length
   no_log: "{{ no_log }}"
 
+- name: Get the current resource task pod information.
+  k8s_info:
+    api_version: v1
+    kind: Pod
+    namespace: '{{ ansible_operator_meta.namespace }}'
+    label_selectors:
+      - "app.kubernetes.io/name={{ ansible_operator_meta.name }}-task"
+      - "app.kubernetes.io/managed-by={{ deployment_type }}-operator"
+      - "app.kubernetes.io/component={{ deployment_type }}"
+    field_selectors:
+      - status.phase=Running
+  register: awx_task_pod
+
+- name: Set the resource pod as a variable.
+  set_fact:
+    awx_task_pod: >-
+      {{ awx_task_pod['resources']
+         | rejectattr('metadata.deletionTimestamp', 'defined')
+         | sort(attribute='metadata.creationTimestamp')
+         | first | default({}) }}
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    awx_task_pod_name: "{{ awx_task_pod['metadata']['name'] | default('') }}"
+
+- name: Precreate database partitions
+  k8s_exec:
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    pod: "{{ awx_task_pod_name }}"
+    container: "{{ deployment_name }}-task"
+    command: awx-manage precreate_partitions --count='{{ precreate_partition_hours }}'
+  when: precreate_partition_hours > 0
+  register: result
+  changed_when: "'Created partitions for' in result.stdout"
+
 - name: Set pg_dump command
   set_fact:
     pgdump: >-

roles/installer/tasks/migrate_data.yml

@@ -76,7 +76,7 @@
       trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
       echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD=\"$PGPASSWORD_OLD\" {{ pgdump }} |  PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      PGPASSWORD=\"$PGPASSWORD_OLD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
       set +e +o pipefail
       echo 'Successful'
       "

roles/installer/tasks/scale_down_deployment.yml

@@ -1,12 +1,14 @@
 ---
-
 - name: Check for presence of Deployment
   k8s_info:
     api_version: apps/v1
     kind: Deployment
-    name: "{{ ansible_operator_meta.name }}"
     namespace: "{{ ansible_operator_meta.namespace }}"
-  register: this_deployment
+    label_selectors:
+      - 'app.kubernetes.io/part-of={{ ansible_operator_meta.name }}'
+      - 'app.kubernetes.io/managed-by={{ deployment_type }}-operator'
+      - 'app.kubernetes.io/component={{ deployment_type }}'
+  register: _deployments
 
 - name: Scale down Deployment for migration
   kubernetes.core.k8s_scale:
@@ -16,8 +18,5 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     replicas: 0
     wait: yes
-    wait_timeout: "{{ termination_grace_period_seconds | default(120) }}"
-  loop:
-    - "{{ ansible_operator_meta.name }}-task"
-    - "{{ ansible_operator_meta.name }}-web"
-  when: this_deployment['resources'] | length
+  loop: "{{ _deployments.resources | map(attribute='metadata.name') | list }}"
+  when: _deployments.resources | length

roles/installer/tasks/upgrade_postgres.yml

@@ -91,11 +91,27 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     pod: "{{ postgres_pod_name }}"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data to new PostgreSQL {{ supported_postgres_version }} Database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD='{{ awx_postgres_pass }}' {{ pgdump }} | PGPASSWORD='{{ awx_postgres_pass }}' {{ pg_restore }}
+      PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   no_log: "{{ no_log }}"
   register: data_migration
   failed_when: "'Successful' not in data_migration.stdout"

roles/restore/tasks/postgres.yml

@@ -50,7 +50,7 @@
   k8s_info:
     api_version: apps/v1
     kind: Deployment
-    name: "{{ ansible_operator_meta.namespace }}-task"
+    name: "{{ deployment_name }}-task"
     namespace: "{{ ansible_operator_meta.namespace }}"
   register: this_deployment
 
@@ -63,8 +63,8 @@
     replicas: 0
     wait: yes
   loop:
-    - "{{ ansible_operator_meta.name }}-task"
-    - "{{ ansible_operator_meta.name }}-web"
+    - "{{ deployment_name }}-task"
+    - "{{ deployment_name }}-web"
   when: this_deployment['resources'] | length
 
 - name: Set full resolvable host name for postgres pod
@@ -87,11 +87,27 @@
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data from old database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
       cat {{ backup_dir }}/tower.db | PGPASSWORD='{{ awx_postgres_pass }}' {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
   failed_when: "'Successful' not in data_migration.stdout"