Adding GitHub check to ensure PRs have the proper X/Y/Z flags

* Bump Postgresql, Nginx and Redis versions
* pg12 --> pg13 upgrade path
* Set supported pg version as a variable to remain DRY
* Make deleting the old db data pvc after upgrade configurable
* Use labels to find the postgres pod

* backup/restore: fix postgres label selector value

We need to use the deployment_name variable for the postgres instance
name.

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>

* backup/restore: add missing default supported_pg_version variable

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>

* restore: update database_host fact with pg suffix

Signed-off-by: Dimitri Savineau <dsavinea@redhat.com>

Co-authored-by: Dimitri Savineau <dsavinea@redhat.com>

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,26 @@
+##### SUMMARY
+<!--- Describe the change, including rationale and design decisions -->
+
+<!---
+If you are fixing an existing issue, please include "fixes #nnn" in your
+commit message and your description; but you should still explain what
+the change does.
+-->
+
+##### ISSUE TYPE
+<!--- Pick one below and delete the rest: -->
+ - Breaking Change 
+ - New or Enhanced Feature
+ - Bug, Docs Fix or other nominal change
+
+##### ADDITIONAL INFORMATION
+<!---
+Include additional information to help people understand the change here.
+For bugs that don't have a linked bug report, a step-by-step reproduction
+of the problem is helpful.
+-->
+
+<!--- Paste verbatim command output below, e.g. before and after your change -->
+```
+
+```

.github/workflows/pr_body_check.yml

@@ -0,0 +1,45 @@
+---
+name: PR Check
+env:
+  BRANCH: ${{ github.base_ref || 'devel' }}
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+jobs:
+  pr-check:
+    name: Scan PR description for semantic versioning keywords
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Write PR body to a file
+        run: |
+          cat >> pr.body << __SOME_RANDOM_PR_EOF__
+          ${{ github.event.pull_request.body }}
+          __SOME_RANDOM_PR_EOF__
+
+      - name: Display the received body for troubleshooting
+        run: cat pr.body
+
+      # We want to write these out individually just incase the options were joined on a single line
+      - name: Check for each of the lines
+        run: |
+          grep "Bug, Docs Fix or other nominal change" pr.body > Z
+          grep "New or Enhanced Feature" pr.body > Y
+          grep "Breaking Change" pr.body > X
+          exit 0
+        # We exit 0 and set the shell to prevent the returns from the greps from failing this step
+        # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
+        shell: bash {0}
+
+      - name: Check for exactly one item
+        run: |
+          if [ $(cat X Y Z | wc -l) != 1 ] ; then
+            echo "The PR body must contain exactly one of [ 'Bug, Docs Fix or other nominal change', 'New or Enhanced Feature', 'Breaking Change' ]"
+            echo "We counted $(cat X Y Z | wc -l)"
+            echo "See the default PR body for examples"
+            exit 255;
+          else
+            exit 0;
+          fi

.helm/starter/templates/postgres-config.yaml

@@ -7,7 +7,7 @@ metadata:
 {{- with $.Values.AWX.postgres }}
 stringData:
   host: {{ .host }}
-  port: {{ .port }}
+  port: {{ .port | quote }}
   database: {{ .dbName }}
   username: {{ .username }}
   password: {{ .password }}

Makefile

@@ -269,7 +269,10 @@ charts:
 	mkdir -p $@
 
 .PHONY: helm-chart
-helm-chart: kustomize helm kubectl-slice yq charts
+helm-chart: helm-chart-generate helm-chart-slice
+
+.PHONY: helm-chart-generate
+helm-chart-generate: kustomize helm kubectl-slice yq charts
 	@echo "== KUSTOMIZE (image and namespace) =="
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 
@@ -292,6 +295,12 @@ helm-chart: kustomize helm kubectl-slice yq charts
 			--output-dir=charts/$(CHART_NAME)/templates \
 			--sort-by-kind
 	@echo "AWX Operator installed with Helm Chart version $(VERSION)" > charts/$(CHART_NAME)/templates/NOTES.txt
+	mkdir charts/$(CHART_NAME)/crds
+	mv charts/$(CHART_NAME)/templates/customresourcedefinition* charts/$(CHART_NAME)/crds
+
+.PHONY: helm-chart-edit
+helm-chart-slice:
+	@echo "== EDIT =="
 	$(foreach file, $(wildcard charts/$(CHART_NAME)/templates/*),$(YQ) -i 'del(.. | select(has("namespace")).namespace)' $(file);)
 	$(foreach file, $(wildcard charts/$(CHART_NAME)/templates/*rolebinding*),$(YQ) -i '.subjects[0].namespace = "{{ .Release.Namespace }}"' $(file);)
 	rm -f charts/$(CHART_NAME)/templates/namespace*.yaml

README.md

@@ -50,6 +50,7 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w
          * [Service Account](#service-account)
       * [Uninstall](#uninstall)
       * [Upgrading](#upgrading)
+         * [Backup](#backup)
          * [v0.14.0](#v0140)
             * [Cluster-scope to Namespace-scope considerations](#cluster-scope-to-namespace-scope-considerations)
             * [Project is now based on v1.x of the operator-sdk project](#project-is-now-based-on-v1x-of-the-operator-sdk-project)
@@ -502,7 +503,7 @@ stringData:
 type: Opaque
 ```
 
-> Please ensure that the value for the variable `password` should _not_ contain single or double quotes (`'`, `"`) or backslashes (`\`) to avoid any issues during deployment, backup or restoration.
+> Please ensure that the value for the variable `password` should _not_ contain single or double quotes (`'`, `"`) or backslashes (`\`) to avoid any issues during deployment, [backup](https://github.com/ansible/awx-operator/tree/devel/roles/backup) or [restoration](https://github.com/ansible/awx-operator/tree/devel/roles/restore).
 
 > It is possible to set a specific username, password, port, or database, but still have the database managed by the operator. In this case, when creating the postgres-configuration secret, the `type: managed` field should be added.
 
@@ -698,8 +699,8 @@ The ability to specify topologySpreadConstraints is also allowed through `topolo
 
 | Name                        | Description                         | Default |
 | --------------------------- | ----------------------------------- | ------- |
-| postgres_image              | Path of the image to pull           | 12      |
-| postgres_image_version      | Image version to pull               | 12      |
+| postgres_image              | Path of the image to pull           | postgres      |
+| postgres_image_version      | Image version to pull               | 13      |
 | node_selector               | AWX pods' nodeSelector              | ''      |
 | topology_spread_constraints | AWX pods' topologySpreadConstraints | ''      |
 | tolerations                 | AWX pods' tolerations               | ''      |
@@ -946,7 +947,7 @@ Example spec file for volumes and volume mounts
 
 In order to register default execution environments from private registries, the Custom Resource needs to know about the pull credentials. Those credentials should be stored as a secret and either specified as `ee_pull_credentials_secret` at the CR spec level, or simply be present on the namespace under the name `<resourcename>-ee-pull-credentials` . Instance initialization will register a `Container registry` type credential on the deployed instance and assign it to the registered default execution environments.
 
-The secret should be formated as follows:
+The secret should be formatted as follows:
 
 ```yaml
 ---
@@ -970,7 +971,7 @@ You can create `image_pull_secret`
 ```
 kubectl create secret <resoucename>-cp-pull-credentials regcred --docker-server=<your-registry-server> --docker-username=<your-name> --docker-password=<your-pword> --docker-email=<your-email>
 ```
-If you need more control (for example, to set a namespace or a label on the new secret) then you can customise the Secret before storing it
+If you need more control (for example, to set a namespace or a label on the new secret) then you can customize the Secret before storing it
 
 Example spec file extra-config
 
@@ -1078,7 +1079,7 @@ Example configuration of `no_log` parameter
 ```
 
 #### Auto upgrade
-With this parameter you can influence the behaviour during an operator upgrade.  
+With this parameter you can influence the behavior during an operator upgrade.  
 If set to `true`, the operator will upgrade the specific instance directly.  
 When the value is set to `false`, and we have a running deployment, the operator will not update the AWX instance.  
 This can be useful when you have multiple AWX instances which you want to upgrade step by step instead of all at once.  
@@ -1142,12 +1143,31 @@ awx.awx.ansible.com "awx-demo" deleted
 
 Deleting an AWX instance will remove all related deployments and statefulsets, however, persistent volumes and secrets will remain. To enforce secrets also getting removed, you can use `garbage_collect_secrets: true`.
 
+**Note**: If you ever intend to recover an AWX from an existing database you will need a copy of the secrets in order to perform a successful recovery.
+
 ### Upgrading
 
 To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX.  To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `image_version` variable in `roles/installer/defaults/main.yml` for that particular release.
 
 Apply the awx-operator.yml for that release to upgrade the operator, and in turn also upgrade your AWX deployment.
 
+#### Backup
+
+The first part of any upgrade should be a backup. Note, there are secrets in the pod which work in conjunction with the database. Having just a database backup without the required secrets will not be sufficient for recovering from an issue when upgrading to a new version. See the [backup role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/backup) for information on how to backup your database and secrets. In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore).
+
+#### PostgreSQL Upgrade Considerations
+
+If there is a PostgreSQL major version upgrade, after the data directory on the PVC is migrated to the new version, the old PVC is kept by default.
+This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically
+after a successful upgrade by setting the following variable on the AWX spec. 
+
+
+```yaml
+  spec:
+    postgres_keep_pvc_after_upgrade: False
+```
+
+
 #### v0.14.0
 
 ##### Cluster-scope to Namespace-scope considerations
@@ -1181,7 +1201,7 @@ Please visit [our contributing guidelines](https://github.com/ansible/awx-operat
 
 ## Release Process
 
-The first step is to create a draft release. Typically this will happen in the [Stage Release](https://github.com/ansible/awx/blob/devel/.github/workflows/stage.yml) workflow for AWX and you dont need to do it as a separate step.
+The first step is to create a draft release. Typically this will happen in the [Stage Release](https://github.com/ansible/awx/blob/devel/.github/workflows/stage.yml) workflow for AWX and you don't need to do it as a separate step.
 
 If you need to do an independent release of the operator, you can run the [Stage Release](https://github.com/ansible/awx-operator/blob/devel/.github/workflows/stage.yml) in the awx-operator repo. Both of these workflows will run smoke tests, so there is no need to do this manually.
 
@@ -1200,7 +1220,7 @@ We ask all of our community members and contributors to adhere to the [Ansible c
 
 ## Get Involved
 
-We welcome your feedback and ideas. The AWX operator uses the same mailing list and IRC chanel as AWX itself. Here's how to reach us with feedback and questions:
+We welcome your feedback and ideas. The AWX operator uses the same mailing list and IRC channel as AWX itself. Here's how to reach us with feedback and questions:
 
 - Join the `#ansible-awx` channel on irc.libera.chat
 - Join the [mailing list](https://groups.google.com/forum/#!forum/awx-project)

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -379,6 +379,9 @@ spec:
                 postgres_selector:
                   description: nodeSelector for the Postgres pods
                   type: string
+                postgres_keep_pvc_after_upgrade:
+                  description: Specify whether or not to keep the old PVC after PostgreSQL upgrades
+                  type: boolean
                 postgres_tolerations:
                   description: node tolerations for the Postgres pods
                   type: string
@@ -493,6 +496,10 @@ spec:
                   description: Should AWX instances be automatically upgraded when operator gets upgraded
                   type: boolean
                   default: true
+                set_self_labels:
+                  description: Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
+                  type: boolean
+                  default: true
               type: object
             status:
               properties:
@@ -515,7 +522,10 @@ spec:
                   description: Secret key secret name of the deployed instance
                   type: string
                 migratedFromSecret:
-                  description: The secret used for migrating an old instance.
+                  description: The secret used for migrating an old instance
+                  type: string
+                upgradedPostgresVersion:
+                  description: Status to indicate that the database has been upgraded to the version in the status
                   type: string
                 version:
                   description: Version of the deployed instance

config/crd/bases/awxbackup.ansible.com_awxbackups.yaml

@@ -32,13 +32,13 @@ spec:
                   description: Name of the deployment to be backed up
                   type: string
                 backup_pvc:
-                  description: Name of the PVC to be used for storing the backup
+                  description: Name of the backup PVC
                   type: string
                 backup_pvc_namespace:
-                  description: Namespace the PVC is in
+                  description: (Deprecated) Namespace the PVC is in
                   type: string
                 backup_storage_requirements:
-                  description: Storage requirements for the PostgreSQL container
+                  description: Storage requirements for backup PVC (may be similar to existing postgres PVC backing up from)
                   type: string
                 backup_storage_class:
                   description: Storage class to use when creating PVC for backup
@@ -58,6 +58,10 @@ spec:
                 no_log:
                   description: Configure no_log for no_log tasks
                   type: string
+                set_self_labels:
+                  description: Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
+                  type: boolean
+                  default: true
             status:
               type: object
               properties:

config/crd/bases/awxrestore.ansible.com_awxrestores.yaml

@@ -33,7 +33,8 @@ spec:
                     - CR
                     - PVC
                 deployment_name:
-                  description: Name of the deployment to be restored to
+                  description: Name of the restored deployment. This should be different from the original deployment name
+                    if the original deployment still exists.
                   type: string
                 backup_name:
                   description: AWXBackup object name
@@ -42,7 +43,7 @@ spec:
                   description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim)
                   type: string
                 backup_pvc_namespace:
-                  description: Namespace the PVC is in
+                  description: (Deprecated) Namespace the PVC is in
                   type: string
                 backup_dir:
                   description: Backup directory name, set as a status found on the awxbackup object (backupDirectory)
@@ -59,6 +60,10 @@ spec:
                 no_log:
                   description: Configure no_log for no_log tasks
                   type: string
+                set_self_labels:
+                  description: Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
+                  type: boolean
+                  default: true
             status:
               type: object
               properties:

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -28,8 +28,7 @@ spec:
       - displayName: Backup persistent volume claim namespace
         path: backup_pvc_namespace
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:text
-        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Backup PVC storage requirements
         path: backup_storage_requirements
         x-descriptors:
@@ -67,12 +66,13 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
       version: v1beta1
-    - description: Restore a previous awx deployment into the namespace
+    - description: Restore a previous awx deployment from an AWXBackup. The 
+        deployment name you provide will be the name of the new AWX CR that will be created.
       displayName: AWX Restore
       kind: AWXRestore
       name: awxrestores.awx.ansible.com
       specDescriptors:
-      - displayName: Backup source to restore ?
+      - displayName: Backup source to restore from
         path: backup_source
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:select:CR
@@ -94,7 +94,7 @@ spec:
       - displayName: Backup namespace
         path: backup_pvc_namespace
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Backup directory in the persistent volume claim
         path: backup_dir
         x-descriptors:
@@ -107,12 +107,10 @@ spec:
       - displayName: PostgreSQL Image
         path: postgres_image
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: PostgreSQL Image Version
         path: postgres_image_version
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
       statusDescriptors:
       - description: The state of the restore
@@ -382,6 +380,10 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Postgres Keep Old Data PVC After Upgrade
+        path: postgres_keep_pvc_after_upgrade
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Postgres Tolerations
         path: postgres_tolerations
         x-descriptors:

roles/backup/README.md

@@ -60,13 +60,7 @@ backup_storage_class: 'standard'
 backup_storage_requirements: '20Gi'
 ```
 
-By default, the backup pvc will be created in the same namespace the awxbackup object is created in.  If you want your backup to be stored
-in a specific namespace, you can do so by specifying `backup_pvc_namespace`.  Keep in mind that you will
-need to provide the same namespace when restoring.  
-
-```
-backup_pvc_namespace: 'custom-namespace'
-```
+The backup pvc will be created in the same namespace the awxbackup object is created in.
 
 If a custom postgres configuration secret was used when deploying AWX, it will automatically be used by the backup role.  
 To check the name of this secret, look at the postgresConfigurationSecret status on your AWX object.  

roles/backup/defaults/main.yml

@@ -22,3 +22,6 @@ finalizer_run: false
 
 # Allow additional parameters to be added to the pg_dump backup command
 pg_dump_suffix: ''
+
+# Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
+set_self_labels: true

roles/backup/tasks/creation.yml

@@ -16,6 +16,7 @@
           app.kubernetes.io/managed-by: "{{ deployment_type }}-operator"
           app.kubernetes.io/component: "{{ deployment_type }}"
           app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}'
+  when: set_self_labels | bool
 
 - name: Look up details for this backup object
   k8s_info:

roles/backup/tasks/postgres.yml

@@ -26,7 +26,7 @@
 - block:
     - name: Delete pod to reload a resource configuration
       set_fact:
-        postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ deployment_name }}"
+        postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ deployment_name }}"
       when: postgres_label_selector is not defined
 
     - name: Get the postgres pod information

roles/backup/vars/main.yml

@@ -1,6 +1,7 @@
 ---
 deployment_type: "awx"
 _postgres_image: postgres
-_postgres_image_version: 12
+_postgres_image_version: 13
 backup_complete: false
 database_type: "unmanaged"
+supported_pg_version: 13

roles/installer/defaults/main.yml

@@ -129,7 +129,7 @@ _image_version: "{{ lookup('env', 'DEFAULT_AWX_VERSION') or 'latest' }}"
 _redis_image: docker.io/redis
 _redis_image_version: 7
 _postgres_image: postgres
-_postgres_image_version: 12
+_postgres_image_version: 13
 _init_container_image: quay.io/centos/centos
 _init_container_image_version: stream8
 image_pull_policy: IfNotPresent
@@ -223,6 +223,9 @@ ee_extra_volume_mounts: ''
 #   kubernetes.io/os: linux
 postgres_selector: ''
 
+# Specify whether or not to keep the old PVC after PostgreSQL upgrades
+postgres_keep_pvc_after_upgrade: True
+
 # Add node tolerations for the Postgres pods.
 # Specify as literal block. E.g.:
 # postgres_tolerations: |
@@ -288,3 +291,6 @@ no_log: 'true'
 # Should AWX instances be automatically upgraded when operator gets upgraded
 #
 auto_upgrade: true
+
+# Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
+set_self_labels: true

roles/installer/tasks/database_configuration.yml

@@ -92,6 +92,62 @@
   set_fact:
     __postgres_configuration_secret: "{{ pg_config['resources'][0]['metadata']['name'] }}"
 
+- name: Store Database Configuration
+  set_fact:
+    awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
+    awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}"
+    awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
+    awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
+    awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
+    awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] |  default('prefer'|b64encode) | b64decode }}"
+  no_log: "{{ no_log }}"
+
+- name: Set database as managed
+  set_fact:
+    managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"
+
+- name: Get the old postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    name: "{{ ansible_operator_meta.name }}-postgres-0"
+    field_selectors:
+      - status.phase=Running
+  register: old_postgres_pod
+
+- name: Look up details for this deployment
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: this_awx
+
+- name: Check if postgres pod is running and version 12
+  block:
+    - name: Set path to PG_VERSION file for given container image
+      set_fact:
+        path_to_pg_version: '{{ postgres_data_path }}/PG_VERSION'
+
+    - name: Get old PostgreSQL version
+      k8s_exec:
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        pod: "{{ ansible_operator_meta.name }}-postgres-0"
+        command: |
+          bash -c """
+          cat {{ path_to_pg_version }}
+          """
+      register: _old_pg_version
+
+    - name: Upgrade data dir from Postgres 12 to 13 if applicable
+      include_tasks: upgrade_postgres.yml
+      when:
+        - _old_pg_version.stdout | default('0') | trim == '12'
+  when:
+    - managed_database
+    - this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default('none') != '12'
+    - old_postgres_pod['resources'] | length  # upgrade is complete and old pg pod has been removed
+
 - block:
     - name: Create Database if no database is specified
       k8s:
@@ -111,7 +167,7 @@
       kubernetes.core.k8s_scale:
         api_version: apps/v1
         kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres"
+        name: "{{ ansible_operator_meta.name }}-postgres-13"
         namespace: "{{ ansible_operator_meta.namespace }}"
         replicas: 0
         wait: yes
@@ -121,7 +177,7 @@
         state: absent
         api_version: apps/v1
         kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres"
+        name: "{{ ansible_operator_meta.name }}-postgres-13"
         namespace: "{{ ansible_operator_meta.namespace }}"
         wait: yes
       when: create_statefulset_result.error == 422
@@ -130,23 +186,29 @@
       k8s:
         apply: true
         definition: "{{ lookup('template', 'postgres.yaml.j2') }}"
-  when: pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed'
+  when: managed_database
 
-- name: Store Database Configuration
+- name: Set Default label selector for custom resource generated postgres
   set_fact:
-    awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
-    awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}"
-    awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
-    awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
-    awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
-    awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] |  default('prefer'|b64encode) | b64decode }}"
-  no_log: "{{ no_log }}"
+    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}"
+  when: postgres_label_selector is not defined
+
+- name: Get the postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+    field_selectors:
+      - status.phase=Running
+  register: postgres_pod
 
 - name: Wait for Database to initialize if managed DB
   k8s_info:
     kind: Pod
     namespace: '{{ ansible_operator_meta.namespace }}'
-    name: '{{ ansible_operator_meta.name }}-postgres-0'  # using name to keep compatibility
+    label_selectors:
+      - "{{ postgres_label_selector }}"
     field_selectors:
       - status.phase=Running
   register: postgres_pod
@@ -156,11 +218,7 @@
     - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
   delay: 5
   retries: 60
-  when: pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed'
-
-- name: Set database as managed
-  set_fact:
-    managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"
+  when: managed_database
 
 - name: Look up details for this deployment
   k8s_info:

roles/installer/tasks/install.yml

@@ -16,6 +16,7 @@
           app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
           app.kubernetes.io/component: '{{ deployment_type }}'
           app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}'
+  when: set_self_labels | bool
 
 - name: Include secret key configuration tasks
   include_tasks: secret_key_configuration.yml

roles/installer/tasks/migrate_data.yml

@@ -13,25 +13,20 @@
     awx_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}"
   no_log: "{{ no_log }}"
 
-- name: Default label selector to custom resource generated postgres
+- name: Set Default label selector for custom resource generated postgres
   set_fact:
-    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ ansible_operator_meta.name }}"
+    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}"
   when: postgres_label_selector is not defined
 
 - name: Get the postgres pod information
   k8s_info:
     kind: Pod
-    namespace: '{{ ansible_operator_meta.namespace }}'
-    name: '{{ ansible_operator_meta.name }}-postgres-0'  # using name to keep compatibility
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+      - "{{ postgres_label_selector }}"
     field_selectors:
       - status.phase=Running
   register: postgres_pod
-  until:
-    - "postgres_pod['resources'] | length"
-    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
-    - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
-  delay: 5
-  retries: 60
 
 - name: Set the resource pod name as a variable.
   set_fact:

roles/installer/tasks/resources_configuration.yml

@@ -11,11 +11,11 @@
       - "app.kubernetes.io/component={{ deployment_type }}"
     field_selectors:
       - status.phase=Running
-  register: tower_pods
+  register: tower_pod
 
 - name: Set the resource pod name as a variable.
   set_fact:
-    tower_pod_name: "{{ tower_pods['resources'][0]['metadata']['name'] | default('') }}"
+    tower_pod_name: "{{ tower_pod['resources'][0]['metadata']['name'] | default('') }}"
 
 - name: Set user provided control plane ee image
   set_fact:
@@ -77,7 +77,7 @@
     apply: yes
     definition: "{{ lookup('template', 'deployment.yaml.j2') }}"
     wait: yes
-  register: tower_deployment_result
+  register: this_deployment_result
 
 - block:
     - name: Delete pod to reload a resource configuration
@@ -113,7 +113,7 @@
       set_fact:
         tower_pod_name: '{{ _new_pod["resources"][0]["metadata"]["name"] }}'
   when:
-    - tower_resources_result.changed or tower_deployment_result.changed
+    - tower_resources_result.changed or this_deployment_result.changed
 
 - name: Verify the resource pod name is populated.
   assert:

roles/installer/tasks/scale_down_deployment.yml

@@ -6,7 +6,7 @@
     kind: Deployment
     name: "{{ ansible_operator_meta.name }}"
     namespace: "{{ ansible_operator_meta.namespace }}"
-  register: tower_deployment
+  register: this_deployment
 
 - name: Scale down Deployment for migration
   kubernetes.core.k8s_scale:
@@ -16,4 +16,4 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     replicas: 0
     wait: yes
-  when: tower_deployment['resources'] | length
+  when: this_deployment['resources'] | length

roles/installer/tasks/update_status.yml

@@ -101,3 +101,13 @@
     status:
       migratedFromSecret: "{{ tower_migrated_from_secret }}"
   when: tower_migrated_from_secret is defined
+
+- name: Update upgradedPostgresVersion status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    status:
+      upgradedPostgresVersion: "{{ upgraded_postgres_version }}"
+  when: upgraded_postgres_version is defined

roles/installer/tasks/upgrade_postgres.yml

@@ -0,0 +1,132 @@
+---
+
+# Upgrade Posgres (Managed Databases only)
+#  * If postgres version is not 12, and not an external postgres instance (when managed_database is yes),
+#    then run this playbook with include_tasks from database_configuration.yml
+#  * Data will be streamed via a pg_dump from the postgres 12 pod to the postgres 13
+#    pod via a pg_restore.
+
+
+- name: Scale down Deployment for migration
+  include_tasks: scale_down_deployment.yml
+
+- name: Delete existing postgres configuration secret
+  k8s:
+    api_version: v1
+    kind: Secret
+    name: "{{ ansible_operator_meta.name }}-postgres-configuration"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    state: absent
+    wait: yes
+
+- name: Create Database configuration with new -postgres-{{ supported_pg_version }} hostname
+  k8s:
+    apply: true
+    definition: "{{ lookup('template', 'postgres_upgrade_secret.yaml.j2') }}"
+  no_log: "{{ no_log }}"
+
+- name: Set new database var to be used when configuring app credentials (resources_configuration.yml)
+  set_fact:
+    awx_postgres_host: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
+  no_log: "{{ no_log }}"
+
+- name: Create Database if no database is specified
+  k8s:
+    apply: true
+    definition: "{{ lookup('template', 'postgres.yaml.j2') }}"
+    wait: true
+  register: create_statefulset_result
+
+- name: Set postgres label if not defined by user
+  set_fact:
+    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}"
+  when: postgres_label_selector is not defined
+
+- name: Get new postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+    field_selectors:
+      - status.phase=Running
+  register: postgres_pod
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+    - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
+  delay: 5
+  retries: 60
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+
+- name: Set full resolvable host name for postgres pod
+  set_fact:
+    resolvable_db_host: "{{ ansible_operator_meta.name }}-postgres.{{ ansible_operator_meta.namespace }}.svc.cluster.local"  # yamllint disable-line rule:line-length
+  no_log: "{{ no_log }}"
+
+- name: Set pg_dump command
+  set_fact:
+    pgdump: >-
+      pg_dump
+      -h {{ resolvable_db_host }}
+      -U {{ awx_postgres_user }}
+      -d {{ awx_postgres_database }}
+      -p {{ awx_postgres_port }}
+      -F custom
+  no_log: "{{ no_log }}"
+
+- name: Set pg_restore command
+  set_fact:
+    pg_restore: >-
+      pg_restore
+      -U {{ awx_postgres_user }}
+      -d {{ awx_postgres_database }}
+  no_log: "{{ no_log }}"
+
+- name: Stream backup from pg_dump to the new postgresql container
+  k8s_exec:
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    pod: "{{ postgres_pod_name }}"
+    command: |
+      bash -c """
+      set -e -o pipefail
+      PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ pg_restore }}
+      echo 'Successful'
+      """
+  no_log: "{{ no_log }}"
+  register: data_migration
+  failed_when: "'Successful' not in data_migration.stdout"
+
+- name: Set flag signifying that this instance has been migrated
+  set_fact:
+    upgraded_postgres_version: '13'
+
+# Cleanup old Postgres resources
+- name: Remove old Postgres StatefulSet
+  k8s:
+    kind: StatefulSet
+    api_version: v1
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    name: "{{ ansible_operator_meta.name }}-postgres"
+    state: absent
+    wait: true
+
+- name: Remove old Postgres Service
+  k8s:
+    kind: Service
+    api_version: v1
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    name: "{{ ansible_operator_meta.name }}-postgres"
+    state: absent
+
+- name: Remove old persistent volume claim
+  k8s:
+    kind: PersistentVolumeClaim
+    api_version: v1
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    name: "postgres-{{ ansible_operator_meta.name }}-postgres-0"
+    state: absent
+  when: postgres_keep_pvc_after_upgrade

roles/installer/templates/postgres.yaml.j2

@@ -3,11 +3,11 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: '{{ ansible_operator_meta.name }}-postgres'
+  name: '{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}'
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: 'postgres'
-    app.kubernetes.io/instance: 'postgres-{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/name: 'postgres-{{ supported_pg_version }}'
+    app.kubernetes.io/instance: 'postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}'
     app.kubernetes.io/component: 'database'
     app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}'
     app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
@@ -16,8 +16,8 @@ metadata:
 spec:
   selector:
     matchLabels:
-      app.kubernetes.io/name: 'postgres'
-      app.kubernetes.io/instance: 'postgres-{{ ansible_operator_meta.name }}'
+      app.kubernetes.io/name: 'postgres-{{ supported_pg_version }}'
+      app.kubernetes.io/instance: 'postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}'
       app.kubernetes.io/component: 'database'
       app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
   serviceName: '{{ ansible_operator_meta.name }}'
@@ -27,8 +27,8 @@ spec:
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: 'postgres'
-        app.kubernetes.io/instance: 'postgres-{{ ansible_operator_meta.name }}'
+        app.kubernetes.io/name: 'postgres-{{ supported_pg_version }}'
+        app.kubernetes.io/instance: 'postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}'
         app.kubernetes.io/component: 'database'
         app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}'
         app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' 
@@ -45,20 +45,6 @@ spec:
 {% if postgres_priority_class is defined %}
       priorityClassName: '{{ postgres_priority_class }}'
 {% endif %}
-      initContainers:
-        - name: database-check
-          image: '{{ _init_container_image }}'
-          resources: {{ postgres_init_container_resource_requirements }}
-          imagePullPolicy: '{{ image_pull_policy }}'
-          command:
-            - /bin/sh
-            - -c
-            - |
-              [[ -d /check-db/pgsql/data ]] && rm -rf /check-db/data && mv /check-db/pgsql/data/ /check-db/data/ && rm -rf /check-db/pgsql || exit 0
-          volumeMounts:
-            - name: postgres
-              mountPath: /check-db
-              subPath: ''
       containers:
         - image: '{{ _postgres_image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -67,7 +53,7 @@ spec:
           args: {{ postgres_extra_args }}
 {% endif %}
           env:
-            # For postgres_image based on rhel8/postgresql-12
+            # For postgres_image based on rhel8/postgresql-13
             - name: POSTGRESQL_DATABASE
               valueFrom:
                 secretKeyRef:
@@ -108,9 +94,9 @@ spec:
               value: '{{ postgres_host_auth_method }}'
           ports:
             - containerPort: {{ awx_postgres_port | default('5432')}}
-              name: postgres
+              name: postgres-{{ supported_pg_version }}
           volumeMounts:
-            - name: postgres
+            - name: postgres-{{ supported_pg_version }}
               mountPath: '{{ postgres_data_path | dirname }}'
               subPath: '{{ postgres_data_path | dirname | basename }}'
           resources: {{ postgres_resource_requirements }}
@@ -124,7 +110,7 @@ spec:
 {% endif %}
   volumeClaimTemplates:
     - metadata:
-        name: postgres
+        name: postgres-{{ supported_pg_version }}
       spec:
         accessModes:
           - ReadWriteOnce
@@ -138,11 +124,11 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: '{{ ansible_operator_meta.name }}-postgres'
+  name: '{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}'
   namespace: '{{ ansible_operator_meta.namespace }}'
   labels:
-    app.kubernetes.io/name: 'postgres'
-    app.kubernetes.io/instance: 'postgres-{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/name: 'postgres-{{ supported_pg_version }}'
+    app.kubernetes.io/instance: 'postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}'
     app.kubernetes.io/component: 'database'
     app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}'
     app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
@@ -153,8 +139,8 @@ spec:
     - port: 5432
   clusterIP: None
   selector:
-    app.kubernetes.io/name: 'postgres'
-    app.kubernetes.io/instance: 'postgres-{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/name: 'postgres-{{ supported_pg_version }}'
+    app.kubernetes.io/instance: 'postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}'
     app.kubernetes.io/component: 'database'
     app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}'
     app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'

roles/installer/templates/postgres_secret.yaml.j2

@@ -16,5 +16,5 @@ stringData:
   username: '{{ database_username }}'
   database: '{{ database_name }}'
   port: '5432'
-  host: {{ ansible_operator_meta.name }}-postgres
+  host: {{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}
   type: 'managed'

roles/installer/templates/postgres_upgrade_secret.yaml.j2

@@ -0,0 +1,20 @@
+# Postgres Secret.
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ ansible_operator_meta.name }}-postgres-configuration'
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
+    app.kubernetes.io/component: '{{ deployment_type }}'
+    app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}'
+stringData:
+  password: '{{ awx_postgres_pass }}'
+  username: '{{ awx_postgres_user }}'
+  database: '{{ awx_postgres_database }}'
+  port: '{{ awx_postgres_port }}'
+  host: '{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}'
+  type: 'managed'

roles/installer/vars/main.yml

@@ -4,3 +4,4 @@ postgres_host_auth_method: 'scram-sha-256'
 ldap_cacert_ca_crt: ''
 bundle_ca_crt: ''
 projects_existing_claim: ''
+supported_pg_version: 13

roles/restore/README.md

@@ -33,7 +33,6 @@ metadata:
 spec:
   deployment_name: mytower
   backup_name: awxbackup-2021-04-22
-  backup_pvc_namespace: 'old-awx-namespace'
 ```
 
 Note that the `deployment_name` above is the name of the AWX deployment you intend to create and restore to.  
@@ -81,11 +80,7 @@ awx-backup-volume-claim
 backup_pvc: 'awx-backup-volume-claim'
 ```
 
-By default, the backup pvc will be created in the same namespace the awxbackup object is created in. This namespace must be specified using the `backup_pvc_namespace` variable.
-
-```
-backup_pvc_namespace: 'custom-namespace'
-```
+The backup pvc will be created in the same namespace the awxbackup object is created in.
 
 If a custom postgres configuration secret was used when deploying AWX, it must be set:
 

roles/restore/defaults/main.yml

@@ -13,3 +13,6 @@ backup_dir: ''
 
 # Set no_log settings on certain tasks
 no_log: 'true'
+
+# Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
+set_self_labels: true

roles/restore/tasks/main.yml

@@ -16,6 +16,7 @@
           app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
           app.kubernetes.io/component: '{{ deployment_type }}'
           app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}'
+  when: set_self_labels | bool
 
 - name: Look up details for this restore object
   k8s_info:

roles/restore/tasks/postgres.yml

@@ -22,28 +22,29 @@
     awx_postgres_type: "{{ pg_config['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}"
   no_log: "{{ no_log }}"
 
-- name: Default label selector to custom resource generated postgres
+- name: Set Default label selector for custom resource generated postgres
   set_fact:
-    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ deployment_name }}"
+    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ deployment_name }}"
   when: postgres_label_selector is not defined
 
-- name: Get the postgres pod information
-  k8s_info:
-    kind: Pod
-    namespace: '{{ ansible_operator_meta.namespace }}'
-    label_selectors:
-      - "{{ postgres_label_selector }}"
-  register: postgres_pod
-  until:
-    - "postgres_pod['resources'] | length"
-    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
-    - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
-  delay: 5
-  retries: 60
-
-- name: Set the resource pod name as a variable.
-  set_fact:
-    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+- block:
+    - name: Get the postgres pod information
+      k8s_info:
+        kind: Pod
+        namespace: '{{ ansible_operator_meta.namespace }}'
+        label_selectors:
+          - "{{ postgres_label_selector }}"
+      register: postgres_pod
+      until:
+        - "postgres_pod['resources'] | length"
+        - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+        - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
+      delay: 5
+      retries: 60
+    - name: Set the resource pod name as a variable.
+      set_fact:
+        postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+  when: awx_postgres_type == 'managed'
 
 - name: Check for presence of AWX Deployment
   k8s_info:

roles/restore/tasks/secrets.yml

@@ -30,7 +30,7 @@
   block:
     - name: Set new database host
       set_fact:
-        database_host: "{{ deployment_name }}-postgres"
+        database_host: "{{ deployment_name }}-postgres-{{ supported_pg_version }}"
       no_log: "{{ no_log }}"
 
     - name: Set tmp postgres secret dict

roles/restore/vars/main.yml

@@ -2,7 +2,7 @@
 
 deployment_type: "awx"
 _postgres_image: postgres
-_postgres_image_version: 12
+_postgres_image_version: 13
 
 backup_api_version: '{{ deployment_type }}.ansible.com/v1beta1'
 backup_kind: 'AWXBackup'
@@ -12,3 +12,4 @@ secret_key_secret: '{{ deployment_name }}-secret-key'
 admin_password_secret: '{{ deployment_name }}-admin-password'
 broadcast_websocket_secret: '{{ deployment_name }}-broadcast-websocket'
 postgres_configuration_secret: '{{ deployment_name }}-postgres-configuration'
+supported_pg_version: 13