Merge pull request #116 from rooftopcellist/backup_restore

Optional data migration if source DB configuration is provided

.ansible-lint

@@ -1,3 +1,4 @@
+---
 skip_list:
   - '306'
   - '602'

README.md

@@ -7,28 +7,31 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w
 # Table of Contents
 
 <!--ts-->
-* [Purpose](#purpose)
-* [Usage](#usage)
-  * [Basic Install](#basic-install)
-  * [Admin user account configuration](#admin-user-account-configuration)
-  * [Network And TLS Configuration](#network-and-tls-configuration)
-    * [Ingress Type](#ingress-type)
-    * [TLS Termination](#tls-termination)
-  * [Database Configuration](#database-configuration)
-    * [External PostgreSQL Service](#external-postgresql-service)
-    * [Managed PostgreSQL Service](#managed-postgresql-service)
-  * [Advanced Configuration](#advanced-configuration)
-    * [Deploying a specific version of AWX](#deploying-a-specific-version-of-awx)
-    * [Privilged Tasks](#privileged-tasks)
-    * [Containers Resource Requirements](#containers-resource-requirements)
-* [Development](#development)
-  * [Testing](#testing)
-    * [Testing in Docker](#testing-in-docker)
-    * [Testing in Minikube](#testing-in-minikube)
-* [Release Process](#release-process)
-  * [Build a new release](#build-a-new-release)
-  * [Build a new version of the operator yaml file](#build-a-new-version-of-the-operator-yaml-file)
-* [Author](#author)
+* [AWX Operator](#awx-operator)
+* [Table of Contents](#table-of-contents)
+   * [Purpose](#purpose)
+   * [Usage](#usage)
+      * [Basic Install](#basic-install)
+      * [Admin user account configuration](#admin-user-account-configuration)
+      * [Network and TLS Configuration](#network-and-tls-configuration)
+         * [Ingress Type](#ingress-type)
+         * [TLS Termination](#tls-termination)
+      * [Database Configuration](#database-configuration)
+         * [External PostgreSQL Service](#external-postgresql-service)
+         * [Migrating data from an old AWX instance](#migrating-data-from-an-old-awx-instance)
+         * [Managed PostgreSQL Service](#managed-postgresql-service)
+      * [Advanced Configuration](#advanced-configuration)
+         * [Deploying a specific version of AWX](#deploying-a-specific-version-of-awx)
+         * [Privileged Tasks](#privileged-tasks)
+         * [Containers Resource Requirements](#containers-resource-requirements)
+   * [Development](#development)
+      * [Testing](#testing)
+         * [Testing in Docker](#testing-in-docker)
+         * [Testing in Minikube](#testing-in-minikube)
+   * [Release Process](#release-process)
+      * [Build a new release](#build-a-new-release)
+      * [Build a new version of the operator yaml file](#build-a-new-version-of-the-operator-yaml-file)
+   * [Author](#author)
 <!--te-->
 
 ## Purpose
@@ -176,6 +179,10 @@ stringData:
 type: Opaque
 ```
 
+#### Migrating data from an old AWX instance
+
+For instructions on how to migrate from an older version of AWX, see [migration.md](./docs/migration.md).
+
 #### Managed PostgreSQL Service
 
 If you don't have access to an external PostgreSQL service, the AWX operator can deploy one for you along side the AWX instance itself.

ansible/instantiate-awx-deployment.yml

@@ -9,7 +9,7 @@
     - name: Deploy AWX
       k8s:
         state: "{{ state | default('present') }}"
-        namespace: default
+        namespace: "{{ namespace | default('default') }}"
         apply: yes
         wait: yes
         definition:

ansible/templates/crd.yml.j2

@@ -44,6 +44,9 @@ spec:
                 tower_postgres_configuration_secret:
                   description: Secret where the database configuration can be found
                   type: string
+                tower_old_postgres_configuration_secret:
+                  description: Secret where the old database configuration can be found for data migration
+                  type: string
                 tower_secret_key_secret:
                   description: Secret where the secret key can be found
                   type: string
@@ -230,6 +233,9 @@ spec:
                 towerAdminPasswordSecret:
                   description: Admin password of the deployed instance
                   type: string
+                towerMigratedFromSecret:
+                  description: The secret used for migrating an old Tower.
+                  type: string
                 towerVersion:
                   description: Version of the deployed instance
                   type: string

ansible/templates/role.yml.j2

@@ -55,6 +55,12 @@ rules:
       - deployments/finalizers
     verbs:
       - update
+  - apiGroups:
+      - apps
+    resources:
+      - deployments/scale
+    verbs:
+      - patch
   - apiGroups:
       - ""
     resources:

deploy/awx-operator.yaml

@@ -57,6 +57,12 @@ rules:
       - deployments/finalizers
     verbs:
       - update
+  - apiGroups:
+      - apps
+    resources:
+      - deployments/scale
+    verbs:
+      - patch
   - apiGroups:
       - ""
     resources:
@@ -116,7 +122,7 @@ spec:
       serviceAccountName: awx-operator
       containers:
         - name: awx-operator
-          image: "quay.io/ansible/awx-operator:0.6.0"
+          image: "quay.io/shanemcd/awx-operator:latest"
           imagePullPolicy: "Always"
           volumeMounts:
             - mountPath: /tmp/ansible-operator/runner
@@ -189,6 +195,9 @@ spec:
                 tower_postgres_configuration_secret:
                   description: Secret where the database configuration can be found
                   type: string
+                tower_old_postgres_configuration_secret:
+                  description: Secret where the old database configuration can be found for data migration
+                  type: string
                 tower_secret_key_secret:
                   description: Secret where the secret key can be found
                   type: string
@@ -375,6 +384,9 @@ spec:
                 towerAdminPasswordSecret:
                   description: Admin password of the deployed instance
                   type: string
+                towerMigratedFromSecret:
+                  description: The secret used for migrating an old Tower.
+                  type: string
                 towerVersion:
                   description: Version of the deployed instance
                   type: string

deploy/crds/awx_v1beta1_crd.yaml

@@ -44,6 +44,9 @@ spec:
                 tower_postgres_configuration_secret:
                   description: Secret where the database configuration can be found
                   type: string
+                tower_old_postgres_configuration_secret:
+                  description: Secret where the old database configuration can be found for data migration
+                  type: string
                 tower_secret_key_secret:
                   description: Secret where the secret key can be found
                   type: string
@@ -230,6 +233,9 @@ spec:
                 towerAdminPasswordSecret:
                   description: Admin password of the deployed instance
                   type: string
+                towerMigratedFromSecret:
+                  description: The secret used for migrating an old Tower.
+                  type: string
                 towerVersion:
                   description: Version of the deployed instance
                   type: string

deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml

@@ -98,6 +98,10 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:Secret
+        path: tower_old_postgres_configuration_secret
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:Secret
       - displayName: Secret key secret
         path: tower_secret_key_secret
         x-descriptors:

deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml

@@ -86,6 +86,9 @@ spec:
               tower_postgres_configuration_secret:
                 description: Secret where the database configuration can be found
                 type: string
+              tower_old_postgres_configuration_secret:
+                description: Secret where the old database configuration can be found for data migration
+                type: string
               tower_postgres_data_path:
                 description: Path where the PostgreSQL data are located
                 type: string

docs/migration.md

@@ -0,0 +1,58 @@
+# Migrating data from an old AWX instance
+
+To migrate data from an older AWX installation, you must provide some information via Secrets.
+
+## Creating Secrets for Migration
+
+### Secret Key
+
+You can find your old secret key in the inventory file you used to deploy AWX in releases prior to version 18. 
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: <resourcename>-secret-key
+  namespace: <target namespace>
+stringData:
+  secret_key: <old-secret-key>
+type: Opaque
+```
+
+**Note**: `<resourcename>` must match the `name` of the AWX object you are creating. In our example below, it is `awx`.
+
+### Old Databse Credentials
+
+The secret should be formatted as follows:
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: <resourcename>-old-postgres-configuration
+  namespace: <target namespace>
+stringData:
+  host: <external ip or url resolvable by the cluster>
+  port: <external port, this usually defaults to 5432>
+  database: <desired database name>
+  username: <username to connect as>
+  password: <password to connect with>
+type: Opaque
+```
+
+> For `host`, a URL resolvable by the cluster could look something like `postgresql.<namespace>.svc.cluster.local`, where `<namespace>` is filled in with the namespace of the AWX deployment you are migrating data from.
+
+## Deploy AWX
+
+When you apply your AWX object, you must specify the name to the database secret you created above:
+
+```yaml
+apiVersion: awx.ansible.com/v1beta1
+kind: AWX
+metadata:
+  name: awx
+spec:
+  tower_old_postgres_configuration_secret: <resourcename>-old-postgres-configuration
+  ...
+```

roles/installer/defaults/main.yml

@@ -1,6 +1,9 @@
 ---
 deployment_type: awx
 
+database_name: "{{ deployment_type }}"
+database_username: "{{ deployment_type }}"
+
 tower_task_privileged: false
 tower_ingress_type: none
 
@@ -47,6 +50,10 @@ tower_broadcast_websocket_secret: ''
 #
 tower_secret_key_secret: ''
 
+# Secret to lookup that provides old database credentials (for migration)
+
+tower_old_postgres_configuration_secret: ''
+
 # Add extra volumes to the AWX pod. Specify as literal block. E.g.:
 # tower_extra_volumes: |
 #   - name: my-volume

roles/installer/tasks/database_configuration.yml

@@ -14,6 +14,22 @@
     name: '{{ meta.name }}-postgres-configuration'
   register: _default_pg_config_resources
 
+- name: Check for old PostgreSQL configuration secret
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: '{{ tower_old_postgres_configuration_secret }}'
+  register: old_pg_config
+  when: tower_old_postgres_configuration_secret | length
+
+- name: Set proper database name when migrating from old deployment
+  set_fact:
+    database_name: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}"
+    database_username: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}"
+  when:
+    - old_pg_config['resources'] is defined
+    - old_pg_config['resources'] | length
+
 - name: Set PostgreSQL configuration
   set_fact:
     _pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}'
@@ -30,7 +46,6 @@
         namespace: '{{ meta.namespace }}'
         name: '{{ meta.name }}-postgres-configuration'
       register: _generated_pg_config_resources
-
   when: not _pg_config['resources'] | default([]) | length
 
 - name: Set PostgreSQL Configuration
@@ -51,3 +66,18 @@
     awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
     awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
     awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
+
+- name: Look up details for this deployment
+  k8s_info:
+    api_version: 'v1beta1'  # TODO: How to parameterize this?
+    kind: "AWX"  #  TODO: How to parameterize this?
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+  register: this_awx
+
+- name: Migrate data from old Openshift instance
+  import_tasks: migrate_data.yml
+  when:
+    - old_pg_config['resources'] is defined
+    - old_pg_config['resources'] | length
+    - this_awx['resources'][0]['status']['towerMigratedFromSecret'] is not defined

roles/installer/tasks/initialize.yml

@@ -23,7 +23,6 @@
       User.objects.create_superuser('{{ tower_admin_user }}', '{{ tower_admin_email }}', '{{ tower_admin_password }}')\"
       | awx-manage shell"
   when: users_result.return_code > 0
-  no_log: true
 
 - name: Create preload data if necessary.  # noqa 305
   community.kubernetes.k8s_exec:

roles/installer/tasks/main.yml

@@ -21,7 +21,6 @@
   k8s:
     apply: yes
     definition: "{{ lookup('template', item) | from_yaml_all | list }}"
-  register: k8s_defs_result
   with_items:
     - tower_config.yaml.j2
 
@@ -57,30 +56,27 @@
     that: tower_pod_name != ''
     fail_msg: "Could not find the tower pod's name."
 
-- name: Check if database is populated (auth_user table exists).
-  community.kubernetes.k8s_exec:
+- name: Check for pending migrations
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"
     command: >-
-      bash -c "echo 'from django.db import connection;
-      tbl = \"auth_user\" in connection.introspection.table_names();
-      exit(0 if tbl else 1)'
-      | awx-manage shell"
-  ignore_errors: true
+      bash -c "awx-manage showmigrations | grep -v '[X]' | grep '[ ]' | wc -l"
   changed_when: false
   register: database_check
-  when: k8s_defs_result is not changed
 
 - name: Migrate the database if the K8s resources were updated.  # noqa 305
-  community.kubernetes.k8s_exec:
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"
     command: >-
       bash -c "awx-manage migrate --noinput"
   register: migrate_result
-  when: (k8s_defs_result is changed) or (database_check is defined and database_check.return_code != 0)
+  when:
+    - database_check is defined
+    - (database_check.stdout|trim) != '0'
 
 - include_tasks: initialize.yml
 

roles/installer/tasks/migrate_data.yml

@@ -0,0 +1,74 @@
+---
+
+- name: Store Database Configuration
+  set_fact:
+    tower_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}"
+    tower_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}"
+    tower_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}"
+    tower_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}"
+    tower_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}"
+
+- name: Get the postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: '{{ meta.namespace }}'
+    label_selectors:
+      - "app={{ deployment_type }}-postgres"
+  register: postgres_pod
+  until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  delay: 5
+  retries: 60
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+
+- name: Check for presence of Deployment
+  k8s_info:
+    api_version: v1
+    kind: Deployment
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+  register: tower_deployment
+
+- name: Scale down Deployment for migration
+  k8s_scale:
+    api_version: v1
+    kind: Deployment
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    replicas: 0
+  when: tower_deployment['resources'] | length
+
+- name: Set pg_dump command
+  set_fact:
+    pgdump: >-
+      pg_dump --clean --create
+      -h {{ tower_old_postgres_host }}
+      -U {{ tower_old_postgres_user }}
+      -d {{ tower_old_postgres_database }}
+      -p {{ tower_old_postgres_port }}
+
+- name: Set pg_restore command
+  set_fact:
+    psql_restore: >-
+      psql -U {{ awx_postgres_user }}
+      -d template1
+      -p {{ awx_postgres_port }}
+
+- name: Stream backup from pg_dump to the new postgresql container
+  community.kubernetes.k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ postgres_pod_name }}"
+    command: |
+      bash -c """
+      set -e -o pipefail
+      PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }}
+      echo 'Successful'
+      """
+  register: data_migration
+  failed_when: "'Successful' not in data_migration.stdout"
+
+- name: Set flag signifying that this instance has been migrated
+  set_fact:
+    tower_migrated_from_secret: "{{ tower_old_postgres_configuration_secret }}"

roles/installer/tasks/update_status.yml

@@ -68,3 +68,13 @@
           towerURL: "https://{{ route_url['resources'][0]['status']['ingress'][0]['host'] }}"
 
   when: tower_ingress_type | lower == 'route'
+
+- name: Update towerMigratedFromSecret status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      towerMigratedFromSecret: "{{ tower_migrated_from_secret }}"
+  when: tower_migrated_from_secret is defined

roles/installer/templates/tower_postgres_secret.yaml.j2

@@ -7,8 +7,8 @@ metadata:
   namespace: '{{ meta.namespace }}'
 stringData:
   password: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}'
-  username: '{{ deployment_type }}'
-  database: '{{ deployment_type }}'
+  username: '{{ database_username }}'
+  database: '{{ database_name }}'
   port: '5432'
   host: {{ meta.name }}-postgres
   type: 'managed'