From 0e1b12f4b1bca0b25890c8921f4d016dc8623807 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 5 Mar 2021 16:41:45 -0500 Subject: [PATCH] Data migration from 3.8 to 4.0 via pg_dump streamed to psql --- README.md | 72 +++++++++++++------ ansible/templates/crd.yml.j2 | 3 + deploy/awx-operator.yaml | 3 + deploy/crds/awx_v1beta1_crd.yaml | 3 + .../manifests/awx.ansible.com_awxs_crd.yaml | 3 + .../tasks/database_configuration.yml | 11 ++- roles/installer/tasks/migrate_data.yml | 17 +++-- 7 files changed, 83 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 8a6c9ca2..c1c4abae 100644 --- a/README.md +++ b/README.md @@ -7,28 +7,31 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w # Table of Contents -* [Purpose](#purpose) -* [Usage](#usage) - * [Basic Install](#basic-install) - * [Admin user account configuration](#admin-user-account-configuration) - * [Network And TLS Configuration](#network-and-tls-configuration) - * [Ingress Type](#ingress-type) - * [TLS Termination](#tls-termination) - * [Database Configuration](#database-configuration) - * [External PostgreSQL Service](#external-postgresql-service) - * [Managed PostgreSQL Service](#managed-postgresql-service) - * [Advanced Configuration](#advanced-configuration) - * [Deploying a specific version of AWX](#deploying-a-specific-version-of-awx) - * [Privilged Tasks](#privileged-tasks) - * [Containers Resource Requirements](#containers-resource-requirements) -* [Development](#development) - * [Testing](#testing) - * [Testing in Docker](#testing-in-docker) - * [Testing in Minikube](#testing-in-minikube) -* [Release Process](#release-process) - * [Build a new release](#build-a-new-release) - * [Build a new version of the operator yaml file](#build-a-new-version-of-the-operator-yaml-file) -* [Author](#author) +* [AWX Operator](#awx-operator) +* [Table of Contents](#table-of-contents) + * [Purpose](#purpose) + * [Usage](#usage) + * [Basic Install](#basic-install) + * [Admin user account configuration](#admin-user-account-configuration) + * [Network and TLS Configuration](#network-and-tls-configuration) + * [Ingress Type](#ingress-type) + * [TLS Termination](#tls-termination) + * [Database Configuration](#database-configuration) + * [External PostgreSQL Service](#external-postgresql-service) + * [Migrating data from an old AWX instance](#migrating-data-from-an-old-awx-instance) + * [Managed PostgreSQL Service](#managed-postgresql-service) + * [Advanced Configuration](#advanced-configuration) + * [Deploying a specific version of AWX](#deploying-a-specific-version-of-awx) + * [Privileged Tasks](#privileged-tasks) + * [Containers Resource Requirements](#containers-resource-requirements) + * [Development](#development) + * [Testing](#testing) + * [Testing in Docker](#testing-in-docker) + * [Testing in Minikube](#testing-in-minikube) + * [Release Process](#release-process) + * [Build a new release](#build-a-new-release) + * [Build a new version of the operator yaml file](#build-a-new-version-of-the-operator-yaml-file) + * [Author](#author) ## Purpose @@ -176,6 +179,31 @@ stringData: type: Opaque ``` +#### Migrating data from an old AWX instance + +To migrate data from the postgresql pod of an AWX deployment in Openshift for Kubernetes, the Custom Resource needs to know about the connection details. Those connection details should be stored as a secret and either specified as `tower_old_postgres_configuration_secret` at the CR spec level, or simply be present on the namespace under the name `-old-postgres-configuration`. + + +The secret should be formatted as follows: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: -old-postgres-configuration + namespace: +stringData: + host: + port: + database: + username: + password: +type: Opaque +``` + +> For `host`, a URL resolvable by the cluster could look something like `postgresql..svc.cluster.local`, where `` is filled in with the namespace of the AWX deployment you are migrating data from. + #### Managed PostgreSQL Service If you don't have access to an external PostgreSQL service, the AWX operator can deploy one for you along side the AWX instance itself. diff --git a/ansible/templates/crd.yml.j2 b/ansible/templates/crd.yml.j2 index 72cb649c..1d1b2733 100644 --- a/ansible/templates/crd.yml.j2 +++ b/ansible/templates/crd.yml.j2 @@ -44,6 +44,9 @@ spec: tower_postgres_configuration_secret: description: Secret where the database configuration can be found type: string + tower_old_postgres_configuration_secret: + description: Secret where the database configuration can be found + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index e7c195ae..8d42b848 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -189,6 +189,9 @@ spec: tower_postgres_configuration_secret: description: Secret where the database configuration can be found type: string + tower_old_postgres_configuration_secret: + description: Secret where the old database configuration can be found for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string diff --git a/deploy/crds/awx_v1beta1_crd.yaml b/deploy/crds/awx_v1beta1_crd.yaml index 72cb649c..08cd4044 100644 --- a/deploy/crds/awx_v1beta1_crd.yaml +++ b/deploy/crds/awx_v1beta1_crd.yaml @@ -44,6 +44,9 @@ spec: tower_postgres_configuration_secret: description: Secret where the database configuration can be found type: string + tower_old_postgres_configuration_secret: + description: Secret where the old database configuration can be found for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string diff --git a/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml b/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml index 52ffaf57..3f4d706a 100644 --- a/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml +++ b/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml @@ -86,6 +86,9 @@ spec: tower_postgres_configuration_secret: description: Secret where the database configuration can be found type: string + tower_old_postgres_configuration_secret: + description: Secret where the old database configuration can be found for data migration + type: string tower_postgres_data_path: description: Path where the PostgreSQL data are located type: string diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index 3690ebc4..342dc8cd 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -52,6 +52,15 @@ awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" # no_log: true #TODO uncomment +- name: Check for old PostgreSQL configuration secret + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_old_postgres_configuration_secret }}' + register: old_pg_config + when: tower_old_postgres_configuration_secret | length + - name: Migrate data from old Openshift instance import_tasks: migrate_data.yml - when: tower_old_postgres_host is defined + when: old_pg_config['resources'][0]['data']['host'] is defined + ignore_errors: true diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index aca925ca..d9845444 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -1,5 +1,13 @@ --- +- name: Store Database Configuration + set_fact: + tower_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}" + tower_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}" + tower_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}" + tower_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}" + tower_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}" + - name: Get the postgres pod information k8s_info: kind: Pod @@ -15,10 +23,9 @@ set_fact: postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" - - name: Set pg_dump command set_fact: - pgdump: | + pgdump: >- pg_dump -h {{ tower_old_postgres_host }} -U {{ tower_old_postgres_user }} -d {{ tower_old_postgres_database }} @@ -26,7 +33,7 @@ - name: Set pg_restore command set_fact: - psql_restore: | + psql_restore: >- psql -U {{ awx_postgres_user }} -d {{ awx_postgres_database }} -p {{ awx_postgres_port }} @@ -36,9 +43,7 @@ namespace: "{{ meta.namespace }}" pod: "{{ postgres_pod_name }}" command: >- - {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} - environment: - PGPASSWORD: "{{ tower_old_postgres_password }}" + bash -c "PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }}" ignore_errors: true register: data_migration # changed_when: success_condition?