Upgrading to PostgreSQL 15 and moving to sclorg images (#1486)

* Upgrading to postgres:15
* Changing image from postgres to sclorg
* Handle scenario where upgrade status is not defined & correct pg tag
* Rework the upgrade logic to be more resiliant for multiple upgrades

---------

Co-authored-by: john-westcott-iv <john-westcott-iv@users.noreply.github.com>
Co-authored-by: Christian M. Adams <chadams@redhat.com>

roles/backup/vars/main.yml

@@ -1,8 +1,8 @@
 ---
 deployment_type: "awx"
-_postgres_image: postgres
-_postgres_image_version: 13
+_postgres_image: quay.io/sclorg/postgresql-15-c9s
+_postgres_image_version: latest
 backup_complete: false
 database_type: "unmanaged"
-supported_pg_version: 13
+supported_pg_version: 15
 image_pull_policy: IfNotPresent

roles/installer/defaults/main.yml

@@ -255,8 +255,8 @@ _image: quay.io/ansible/awx
 _image_version: "{{ lookup('env', 'DEFAULT_AWX_VERSION') or 'latest' }}"
 _redis_image: docker.io/redis
 _redis_image_version: 7
-_postgres_image: postgres
-_postgres_image_version: 13
+_postgres_image: quay.io/sclorg/postgresql-15-c9s
+_postgres_image_version: latest
 image_pull_policy: IfNotPresent
 image_pull_secrets: []
 

roles/installer/tasks/database_configuration.yml

@@ -106,14 +106,38 @@
   set_fact:
     managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"
 
-- name: Get the old postgres pod information
+# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades.
+# So we have to take that into account and preferentially set the most recent one.
+- name: Get the old postgres pod (N-1)
   k8s_info:
     kind: Pod
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ ansible_operator_meta.name }}-postgres-0"
     field_selectors:
       - status.phase=Running
-  register: old_postgres_pod
+  register: _running_pods
+
+- block:
+  - name: Filter pods by name
+    set_fact:
+      filtered_old_postgres_pods: "{{ _running_pods.resources |
+        selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
+        rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') |
+        list }}"
+
+  # Sort pods by name in reverse order (most recent PG version first) and set
+  - name: Set info for previous postgres pod
+    set_fact:
+      sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
+        sort(attribute='metadata.name') |
+        reverse }}"
+    when: filtered_old_postgres_pods | length
+
+
+  - name: Set info for previous postgres pod
+    set_fact:
+      old_postgres_pod: "{{ sorted_old_postgres_pods | first }}"
+    when: filtered_old_postgres_pods | length
+  when: _running_pods.resources | length
 
 - name: Look up details for this deployment
   k8s_info:
@@ -123,7 +147,14 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
   register: this_awx
 
-- name: Check if postgres pod is running and version 12
+# If this deployment has been upgraded before or if upgrade has already been started, set this var
+- name: Set previous PG version var
+  set_fact:
+    _previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
+  when:
+    - "'upgradedPostgresVersion' in this_awx['resources'][0]['status']"
+
+- name: Check if postgres pod is running an older version
   block:
     - name: Set path to PG_VERSION file for given container image
       set_fact:
@@ -132,21 +163,24 @@
     - name: Get old PostgreSQL version
       k8s_exec:
         namespace: "{{ ansible_operator_meta.namespace }}"
-        pod: "{{ ansible_operator_meta.name }}-postgres-0"
+        pod: "{{ old_postgres_pod['metadata']['name'] }}"
         command: |
           bash -c """
           cat {{ path_to_pg_version }}
           """
       register: _old_pg_version
 
-    - name: Upgrade data dir from Postgres 12 to 13 if applicable
+    - debug:
+        msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"
+
+    - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
       include_tasks: upgrade_postgres.yml
       when:
-        - _old_pg_version.stdout | default('0') | trim == '12'
+        - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
   when:
     - managed_database
-    - this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default('none') != '12'
-    - old_postgres_pod['resources'] | length  # upgrade is complete and old pg pod has been removed
+    - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version < supported_pg_version, true)
+    - old_postgres_pod | length  # If empty, then old pg pod has been removed and we can assume the upgrade is complete
 
 - block:
     - name: Create Database if no database is specified
@@ -167,7 +201,7 @@
       kubernetes.core.k8s_scale:
         api_version: apps/v1
         kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres-13"
+        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
         namespace: "{{ ansible_operator_meta.namespace }}"
         replicas: 0
         wait: yes
@@ -177,7 +211,7 @@
         state: absent
         api_version: apps/v1
         kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres-13"
+        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
         namespace: "{{ ansible_operator_meta.namespace }}"
         wait: yes
       when: create_statefulset_result.error == 422

roles/installer/tasks/update_status.yml

@@ -111,5 +111,5 @@
     name: "{{ ansible_operator_meta.name }}"
     namespace: "{{ ansible_operator_meta.namespace }}"
     status:
-      upgradedPostgresVersion: "{{ upgraded_postgres_version }}"
+      upgradedPostgresVersion: "{{ upgraded_postgres_version | string }}"
   when: upgraded_postgres_version is defined

roles/installer/tasks/upgrade_postgres.yml

@@ -1,9 +1,9 @@
 ---
 
 # Upgrade Posgres (Managed Databases only)
-#  * If postgres version is not 12, and not an external postgres instance (when managed_database is yes),
+#  * If postgres version is not supported_pg_version, and not an external postgres instance (when managed_database is yes),
 #    then run this playbook with include_tasks from database_configuration.yml
-#  * Data will be streamed via a pg_dump from the postgres 12 pod to the postgres 13
+#  * Data will be streamed via a pg_dump from the postgres 12/13 pod to the postgres supported_pg_version
 #    pod via a pg_restore.
 
 
@@ -62,9 +62,19 @@
   set_fact:
     postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
 
+- name: Get the name of the service for the old postgres pod
+  k8s_info:
+    kind: Service
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+      - "app.kubernetes.io/component=database"
+      - "app.kubernetes.io/instance={{ old_postgres_pod.metadata.labels['app.kubernetes.io/instance'] }}"
+      - "app.kubernetes.io/managed-by=awx-operator"
+  register: old_postgres_svc
+
 - name: Set full resolvable host name for postgres pod
   set_fact:
-    resolvable_db_host: "{{ ansible_operator_meta.name }}-postgres.{{ ansible_operator_meta.namespace }}.svc"  # yamllint disable-line rule:line-length
+    resolvable_db_host: "{{ old_postgres_svc['resources'][0]['metadata']['name'] }}.{{ ansible_operator_meta.namespace }}.svc"  # yamllint disable-line rule:line-length
   no_log: "{{ no_log }}"
 
 - name: Set pg_dump command
@@ -118,7 +128,7 @@
 
 - name: Set flag signifying that this instance has been migrated
   set_fact:
-    upgraded_postgres_version: '13'
+    upgraded_postgres_version: '{{ supported_pg_version }}'
 
 # Cleanup old Postgres resources
 - name: Remove old Postgres StatefulSet
@@ -126,23 +136,32 @@
     kind: StatefulSet
     api_version: v1
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ ansible_operator_meta.name }}-postgres"
+    name: "{{ item }}"
     state: absent
     wait: true
+  loop:
+    - "{{ ansible_operator_meta.name }}-postgres"
+    - "{{ ansible_operator_meta.name }}-postgres-13"
 
 - name: Remove old Postgres Service
   k8s:
     kind: Service
     api_version: v1
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ ansible_operator_meta.name }}-postgres"
+    name: "{{ item }}"
     state: absent
+  loop:
+    - "{{ ansible_operator_meta.name }}-postgres"
+    - "{{ ansible_operator_meta.name }}-postgres-13"
 
 - name: Remove old persistent volume claim
   k8s:
     kind: PersistentVolumeClaim
     api_version: v1
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "postgres-{{ ansible_operator_meta.name }}-postgres-0"
+    name: "{{ item }}"
     state: absent
+  loop:
+    - "postgres-{{ ansible_operator_meta.name }}-postgres-0"
+    - "postgres-{{ ansible_operator_meta.name }}-postgres-13-0"
   when: postgres_keep_pvc_after_upgrade

roles/installer/templates/statefulsets/postgres.yaml.j2

@@ -59,7 +59,7 @@ spec:
           args: {{ postgres_extra_args }}
 {% endif %}
           env:
-            # For postgres_image based on rhel8/postgresql-13
+            # For postgres_image based on rhel8/postgresql-{{ supported_pg_version }}
             - name: POSTGRESQL_DATABASE
               valueFrom:
                 secretKeyRef:

roles/installer/vars/main.yml

@@ -4,4 +4,6 @@ postgres_host_auth_method: 'scram-sha-256'
 ldap_cacert_ca_crt: ''
 bundle_ca_crt: ''
 projects_existing_claim: ''
-supported_pg_version: 13
+supported_pg_version: 15
+_previous_upgraded_pg_version: 0
+old_postgres_pod: []

roles/restore/README.md

@@ -19,7 +19,7 @@ This role assumes you are authenticated with an Openshift or Kubernetes cluster:
 
 *Before Restoring from a backup*, be sure to:
   - delete the old existing AWX CR
-  - delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-13-<deployment-name>-postgres-13-0`
+  - delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-<postgres version>-<deployment-name>-postgres-<postgres version>-0`
 
 **Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well.
 

roles/restore/vars/main.yml

@@ -1,8 +1,8 @@
 ---
 
 deployment_type: "awx"
-_postgres_image: postgres
-_postgres_image_version: 13
+_postgres_image: quay.io/sclorg/postgresql-15-c9s
+_postgres_image_version: latest
 
 backup_api_version: '{{ deployment_type }}.ansible.com/v1beta1'
 backup_kind: 'AWXBackup'
@@ -12,7 +12,7 @@ secret_key_secret: '{{ deployment_name }}-secret-key'
 admin_password_secret: '{{ deployment_name }}-admin-password'
 broadcast_websocket_secret: '{{ deployment_name }}-broadcast-websocket'
 postgres_configuration_secret: '{{ deployment_name }}-postgres-configuration'
-supported_pg_version: 13
+supported_pg_version: 15
 image_pull_policy: IfNotPresent
 
 # If set to true, the restore process will delete the existing database and create a new one