Fix reconcilation loop after data migration changes

ansible/templates/crd.yml.j2

@@ -45,7 +45,7 @@ spec:
                   description: Secret where the database configuration can be found
                   type: string
                 tower_old_postgres_configuration_secret:
-                  description: Secret where the database configuration can be found
+                  description: Secret where the old database configuration can be found for data migration
                   type: string
                 tower_secret_key_secret:
                   description: Secret where the secret key can be found
@@ -233,6 +233,9 @@ spec:
                 towerAdminPasswordSecret:
                   description: Admin password of the deployed instance
                   type: string
+                towerMigratedFromSecret:
+                  description: The secret used for migrating an old Tower.
+                  type: string
                 towerVersion:
                   description: Version of the deployed instance
                   type: string

ansible/templates/role.yml.j2

@@ -55,6 +55,12 @@ rules:
       - deployments/finalizers
     verbs:
       - update
+  - apiGroups:
+      - apps
+    resources:
+      - deployments/scale
+    verbs:
+      - patch
   - apiGroups:
       - ""
     resources:

deploy/awx-operator.yaml

@@ -57,6 +57,12 @@ rules:
       - deployments/finalizers
     verbs:
       - update
+  - apiGroups:
+      - apps
+    resources:
+      - deployments/scale
+    verbs:
+      - patch
   - apiGroups:
       - ""
     resources:
@@ -116,7 +122,7 @@ spec:
       serviceAccountName: awx-operator
       containers:
         - name: awx-operator
-          image: "quay.io/ansible/awx-operator:0.6.0"
+          image: "quay.io/shanemcd/awx-operator:latest"
           imagePullPolicy: "Always"
           volumeMounts:
             - mountPath: /tmp/ansible-operator/runner
@@ -378,6 +384,9 @@ spec:
                 towerAdminPasswordSecret:
                   description: Admin password of the deployed instance
                   type: string
+                towerMigratedFromSecret:
+                  description: The secret used for migrating an old Tower.
+                  type: string
                 towerVersion:
                   description: Version of the deployed instance
                   type: string

deploy/crds/awx_v1beta1_crd.yaml

@@ -233,6 +233,9 @@ spec:
                 towerAdminPasswordSecret:
                   description: Admin password of the deployed instance
                   type: string
+                towerMigratedFromSecret:
+                  description: The secret used for migrating an old Tower.
+                  type: string
                 towerVersion:
                   description: Version of the deployed instance
                   type: string

roles/installer/defaults/main.yml

@@ -47,6 +47,10 @@ tower_broadcast_websocket_secret: ''
 #
 tower_secret_key_secret: ''
 
+# Secret to lookup that provides old database credentials (for migration)
+
+tower_old_postgres_configuration_secret: ''
+
 # Add extra volumes to the AWX pod. Specify as literal block. E.g.:
 # tower_extra_volumes: |
 #   - name: my-volume

roles/installer/tasks/database_configuration.yml

@@ -58,6 +58,14 @@
     awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
   no_log: true
 
+- name: Check to see if this instance has already been migrated
+  k8s_info:
+    api_version: 'v1beta1' # TODO: How to parameterize this?
+    kind: "AWX" # TODO: How to parameterize this?
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+  register: this_awx
+
 - name: Check for old PostgreSQL configuration secret
   k8s_info:
     kind: Secret
@@ -65,8 +73,9 @@
     name: '{{ tower_old_postgres_configuration_secret }}'
   register: old_pg_config
   when: tower_old_postgres_configuration_secret | length
-  no_log: true
 
 - name: Migrate data from old Openshift instance
   import_tasks: migrate_data.yml
-  when: old_pg_config['resources'][0]['data']['host'] is defined
+  when:
+    - old_pg_config['resources'][0]['data']['host'] is defined
+    - this_awx['resources'][0]['status']['towerMigratedFromSecret'] is not defined

roles/installer/tasks/main.yml

@@ -21,7 +21,6 @@
   k8s:
     apply: yes
     definition: "{{ lookup('template', item) | from_yaml_all | list }}"
-  register: k8s_defs_result
   with_items:
     - tower_config.yaml.j2
 
@@ -57,29 +56,29 @@
     that: tower_pod_name != ''
     fail_msg: "Could not find the tower pod's name."
 
-- name: Check if database is populated (auth_user table exists).
-  community.kubernetes.k8s_exec:
+- name: Check for pending migrations
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"
     command: >-
       bash -c "awx-manage showmigrations | grep -v '[X]' | grep '[ ]' | wc -l"
-  ignore_errors: true
   changed_when: false
   register: database_check
-  when: k8s_defs_result is not changed
 
 - name: Migrate the database if the K8s resources were updated.  # noqa 305
-  community.kubernetes.k8s_exec:
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"
     command: >-
       bash -c "awx-manage migrate --noinput"
   register: migrate_result
-  when: (k8s_defs_result is changed) or (database_check is defined and database_check.stdout != 0)
+  when:
+    - database_check is defined
+    - (database_check.stdout|trim) != '0'
 
 - include_tasks: initialize.yml
 
-- name: Update status variables
-  include_tasks: update_status.yml
+# - name: Update status variables
+#   include_tasks: update_status.yml

roles/installer/tasks/migrate_data.yml

@@ -24,13 +24,22 @@
   set_fact:
     postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
 
-- name: Scale deployment down when current replicas match
-  community.kubernetes.k8s_scale:
+- name: Check for presence of Deployment
+  k8s_info:
+    api_version: v1
+    kind: Deployment
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+  register: tower_deployment
+
+- name: Scale down Deployment for migration
+  k8s_scale:
     api_version: v1
     kind: Deployment
     name: "{{ meta.name }}"
     namespace: "{{ meta.namespace }}"
     replicas: 0
+  when: tower_deployment['resources'] | length
 
 - name: Set pg_dump command
   set_fact:
@@ -53,7 +62,10 @@
     namespace: "{{ meta.namespace }}"
     pod: "{{ postgres_pod_name }}"
     command: >-
-      bash -c "PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} && echo 'Finished'"
-  no_log: true
+      bash -c "set -o pipefail; PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} && echo 'Successful'"
   register: data_migration
-  changed_when: false
+  failed_when: "'Successful' not in data_migration.stdout"
+
+- name: Set flag signifying that this instance has been migrated
+  set_fact:
+    tower_migrated_from_secret: "{{ tower_old_postgres_configuration_secret }}"

roles/installer/tasks/update_status.yml

@@ -68,3 +68,13 @@
           towerURL: "https://{{ route_url['resources'][0]['status']['ingress'][0]['host'] }}"
 
   when: tower_ingress_type | lower == 'route'
+
+- name: Update towerMigratedFromSecret status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      towerMigratedFromSecret: "{{ tower_migrated_from_secret }}"
+  when: tower_migrated_from_secret is defined