Data migration from 3.8 to 4.0 via pg_dump streamed to psql

2026-03-26 21:33:14 +00:00 · 2021-03-05 16:41:45 -05:00
parent 93d53c712c
commit 0e1b12f4b1
7 changed files with 83 additions and 29 deletions
--- a/README.md
+++ b/README.md
@@ -7,28 +7,31 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w
 # Table of Contents

 <!--ts-->
-* [Purpose](#purpose)
-* [Usage](#usage)
-  * [Basic Install](#basic-install)
-  * [Admin user account configuration](#admin-user-account-configuration)
-  * [Network And TLS Configuration](#network-and-tls-configuration)
-    * [Ingress Type](#ingress-type)
-    * [TLS Termination](#tls-termination)
-  * [Database Configuration](#database-configuration)
-    * [External PostgreSQL Service](#external-postgresql-service)
-    * [Managed PostgreSQL Service](#managed-postgresql-service)
-  * [Advanced Configuration](#advanced-configuration)
-    * [Deploying a specific version of AWX](#deploying-a-specific-version-of-awx)
-    * [Privilged Tasks](#privileged-tasks)
-    * [Containers Resource Requirements](#containers-resource-requirements)
-* [Development](#development)
-  * [Testing](#testing)
-    * [Testing in Docker](#testing-in-docker)
-    * [Testing in Minikube](#testing-in-minikube)
-* [Release Process](#release-process)
-  * [Build a new release](#build-a-new-release)
-  * [Build a new version of the operator yaml file](#build-a-new-version-of-the-operator-yaml-file)
-* [Author](#author)
+* [AWX Operator](#awx-operator)
+* [Table of Contents](#table-of-contents)
+   * [Purpose](#purpose)
+   * [Usage](#usage)
+      * [Basic Install](#basic-install)
+      * [Admin user account configuration](#admin-user-account-configuration)
+      * [Network and TLS Configuration](#network-and-tls-configuration)
+         * [Ingress Type](#ingress-type)
+         * [TLS Termination](#tls-termination)
+      * [Database Configuration](#database-configuration)
+         * [External PostgreSQL Service](#external-postgresql-service)
+         * [Migrating data from an old AWX instance](#migrating-data-from-an-old-awx-instance)
+         * [Managed PostgreSQL Service](#managed-postgresql-service)
+      * [Advanced Configuration](#advanced-configuration)
+         * [Deploying a specific version of AWX](#deploying-a-specific-version-of-awx)
+         * [Privileged Tasks](#privileged-tasks)
+         * [Containers Resource Requirements](#containers-resource-requirements)
+   * [Development](#development)
+      * [Testing](#testing)
+         * [Testing in Docker](#testing-in-docker)
+         * [Testing in Minikube](#testing-in-minikube)
+   * [Release Process](#release-process)
+      * [Build a new release](#build-a-new-release)
+      * [Build a new version of the operator yaml file](#build-a-new-version-of-the-operator-yaml-file)
+   * [Author](#author)
 <!--te-->

 ## Purpose
@@ -176,6 +179,31 @@ stringData:
 type: Opaque
 ```

+#### Migrating data from an old AWX instance
+
+To migrate data from the postgresql pod of an AWX deployment in Openshift for Kubernetes, the Custom Resource needs to know about the connection details. Those connection details should be stored as a secret and either specified as `tower_old_postgres_configuration_secret` at the CR spec level, or simply be present on the namespace under the name `<resourcename>-old-postgres-configuration`.
+
+
+The secret should be formatted as follows:
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: <resourcename>-old-postgres-configuration
+  namespace: <target namespace>
+stringData:
+  host: <external ip or url resolvable by the cluster>
+  port: <external port, this usually defaults to 5432>
+  database: <desired database name>
+  username: <username to connect as>
+  password: <password to connect with>
+type: Opaque
+```
+
+> For `host`, a URL resolvable by the cluster could look something like `postgresql.<namespace>.svc.cluster.local`, where `<namespace>` is filled in with the namespace of the AWX deployment you are migrating data from.  
+
 #### Managed PostgreSQL Service

 If you don't have access to an external PostgreSQL service, the AWX operator can deploy one for you along side the AWX instance itself.
--- a/ansible/templates/crd.yml.j2
+++ b/ansible/templates/crd.yml.j2
@@ -44,6 +44,9 @@ spec:
                tower_postgres_configuration_secret:
                  description: Secret where the database configuration can be found
                  type: string
+                tower_old_postgres_configuration_secret:
+                  description: Secret where the database configuration can be found
+                  type: string
                tower_secret_key_secret:
                  description: Secret where the secret key can be found
                  type: string
--- a/deploy/awx-operator.yaml
+++ b/deploy/awx-operator.yaml
@@ -189,6 +189,9 @@ spec:
                tower_postgres_configuration_secret:
                  description: Secret where the database configuration can be found
                  type: string
+                tower_old_postgres_configuration_secret:
+                  description: Secret where the old database configuration can be found for data migration
+                  type: string
                tower_secret_key_secret:
                  description: Secret where the secret key can be found
                  type: string
--- a/deploy/crds/awx_v1beta1_crd.yaml
+++ b/deploy/crds/awx_v1beta1_crd.yaml
@@ -44,6 +44,9 @@ spec:
                tower_postgres_configuration_secret:
                  description: Secret where the database configuration can be found
                  type: string
+                tower_old_postgres_configuration_secret:
+                  description: Secret where the old database configuration can be found for data migration
+                  type: string
                tower_secret_key_secret:
                  description: Secret where the secret key can be found
                  type: string
--- a/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml
+++ b/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml
@@ -86,6 +86,9 @@ spec:
              tower_postgres_configuration_secret:
                description: Secret where the database configuration can be found
                type: string
+              tower_old_postgres_configuration_secret:
+                description: Secret where the old database configuration can be found for data migration
+                type: string
              tower_postgres_data_path:
                description: Path where the PostgreSQL data are located
                type: string
--- a/roles/installer/tasks/database_configuration.yml
+++ b/roles/installer/tasks/database_configuration.yml
@@ -52,6 +52,15 @@
    awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
  # no_log: true #TODO uncomment

+- name: Check for old PostgreSQL configuration secret
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: '{{ tower_old_postgres_configuration_secret }}'
+  register: old_pg_config
+  when: tower_old_postgres_configuration_secret | length
+
 - name: Migrate data from old Openshift instance
  import_tasks: migrate_data.yml
-  when: tower_old_postgres_host is defined
+  when:  old_pg_config['resources'][0]['data']['host'] is defined
+  ignore_errors: true
--- a/roles/installer/tasks/migrate_data.yml
+++ b/roles/installer/tasks/migrate_data.yml
@@ -1,5 +1,13 @@
 ---

+- name: Store Database Configuration
+  set_fact:
+    tower_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}"
+    tower_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}"
+    tower_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}"
+    tower_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}"
+    tower_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}"
+
 - name: Get the postgres pod information
  k8s_info:
    kind: Pod
@@ -15,10 +23,9 @@
  set_fact:
    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"

-
 - name: Set pg_dump command
  set_fact: 
-    pgdump: |
+    pgdump: >-
      pg_dump -h {{ tower_old_postgres_host }}
      -U {{ tower_old_postgres_user }}
      -d {{ tower_old_postgres_database }}
@@ -26,7 +33,7 @@

 - name: Set pg_restore command
  set_fact: 
-    psql_restore: |
+    psql_restore: >-
      psql -U {{ awx_postgres_user }} 
      -d {{ awx_postgres_database }}
      -p {{ awx_postgres_port }}
@@ -36,9 +43,7 @@
    namespace: "{{ meta.namespace }}"
    pod: "{{ postgres_pod_name }}"
    command: >-
-      {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }}
-  environment:
-    PGPASSWORD: "{{ tower_old_postgres_password }}"
+      bash -c "PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }}"
  ignore_errors: true
  register: data_migration
  # changed_when: success_condition?