From 5b32c412770c602f9320873f3b2921377b3ad6df Mon Sep 17 00:00:00 2001
From: "Christian M. Adams" <rooftopcellist@gmail.com>
Date: Tue, 13 Apr 2021 14:47:38 -0400
Subject: [PATCH] Fix retry for checking postgres pod & fix secrets template  -
 fixed a lot of typos & updated the README.md files

---
 ansible/templates/awxrestore_crd.yml.j2       |  6 +--
 ansible/templates/role.yml.j2                 |  1 +
 deploy/awx-operator.yaml                      | 11 +++---
 deploy/crds/awxrestore_v1beta1_crd.yaml       |  8 ++--
 roles/backup/README.md                        | 24 +++++++-----
 roles/backup/meta/main.yml                    |  1 +
 roles/backup/tasks/init.yml                   |  2 +-
 roles/backup/tasks/main.yml                   |  2 +-
 roles/backup/tasks/postgres.yml               |  6 ++-
 roles/backup/tasks/secrets.yml                | 10 +----
 .../templates/admin_password_secret.yml.j2    | 10 -----
 .../broadcast_websocket_secret.yml.j2         | 10 -----
 roles/backup/templates/postgres_secret.yml.j2 | 16 --------
 .../backup/templates/secret_key_secret.yml.j2 | 10 -----
 roles/restore/README.md                       | 12 +++---
 roles/restore/meta/main.yml                   |  1 +
 roles/restore/tasks/apply_secret.yml          | 24 ------------
 .../tasks/{init_awx.yml => deploy_awx.yml}    | 11 +++---
 roles/restore/tasks/init_secrets.yml          | 19 ----------
 roles/restore/tasks/main.yml                  |  6 +--
 roles/restore/tasks/postgres.yml              |  6 ++-
 roles/restore/tasks/secrets.yml               | 37 +++++++++++++++++++
 roles/restore/templates/awx_object.yml.j2     |  2 +-
 roles/restore/templates/secrets.yml.j2        |  6 ---
 24 files changed, 97 insertions(+), 144 deletions(-)
 delete mode 100644 roles/backup/templates/admin_password_secret.yml.j2
 delete mode 100644 roles/backup/templates/broadcast_websocket_secret.yml.j2
 delete mode 100644 roles/backup/templates/postgres_secret.yml.j2
 delete mode 100644 roles/backup/templates/secret_key_secret.yml.j2
 delete mode 100644 roles/restore/tasks/apply_secret.yml
 rename roles/restore/tasks/{init_awx.yml => deploy_awx.yml} (74%)
 delete mode 100644 roles/restore/tasks/init_secrets.yml
 create mode 100644 roles/restore/tasks/secrets.yml

diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2
index e0ba2070..07f59d48 100644
--- a/ansible/templates/awxrestore_crd.yml.j2
+++ b/ansible/templates/awxrestore_crd.yml.j2
@@ -21,7 +21,7 @@ spec:
         openAPIV3Schema:
           type: object
           x-kubernetes-preserve-unknown-fields: true
-          description: Schema validation for the AWXBackup CRD
+          description: Schema validation for the AWXRestore CRD
           properties:
             spec:
               type: object
@@ -30,10 +30,10 @@ spec:
                   description: Name of the deployment to be restored to
                   type: string
                 tower_backup_pvc:
-                  description: Name of the PVC to be used for storing the backup
+                  description: Name of the PVC to be restored from
                   type: string
                 tower_backup_pvc_namespace:
-                  description: Namespace PVC is in
+                  description: Namespace the PVC is in
                   type: string
                 tower_backup_dir:
                   description: Backup directory name, a status found on the awxbackup object (towerBackupComplete)
diff --git a/ansible/templates/role.yml.j2 b/ansible/templates/role.yml.j2
index 80405470..886bd5f0 100644
--- a/ansible/templates/role.yml.j2
+++ b/ansible/templates/role.yml.j2
@@ -80,5 +80,6 @@ rules:
     resources:
       - '*'
       - awxbackups
+      - awxrestores
     verbs:
       - '*'
diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml
index d5119824..23325e41 100644
--- a/deploy/awx-operator.yaml
+++ b/deploy/awx-operator.yaml
@@ -457,7 +457,7 @@ spec:
         openAPIV3Schema:
           type: object
           x-kubernetes-preserve-unknown-fields: true
-          description: Schema validation for the AWXBackup CRD
+          description: Schema validation for the AWXRestore CRD
           properties:
             spec:
               type: object
@@ -466,10 +466,10 @@ spec:
                   description: Name of the deployment to be restored to
                   type: string
                 tower_backup_pvc:
-                  description: Name of the PVC to be used for storing the backup
+                  description: Name of the PVC to be restored from
                   type: string
                 tower_backup_pvc_namespace:
-                  description: Namespace PVC is in
+                  description: Namespace the PVC is in
                   type: string
                 tower_backup_dir:
                   description: Backup directory name, a status found on the awxbackup object (towerBackupComplete)
@@ -487,7 +487,7 @@ spec:
                   description: Custom postgres_configuration secret name
                   type: string
               oneOf:
-                - required: ["tower_name", "tower_backup_pvc"]
+                - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -570,6 +570,7 @@ rules:
     resources:
       - '*'
       - awxbackups
+      - awxrestores
     verbs:
       - '*'
 
@@ -612,7 +613,7 @@ spec:
       serviceAccountName: awx-operator
       containers:
         - name: awx-operator
-          image: "quay.io/chadams/awx-operator:new-crd"
+          image: "quay.io/ansible/awx-operator:0.8.0"
           imagePullPolicy: "Always"
           volumeMounts:
             - mountPath: /tmp/ansible-operator/runner
diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml
index e7e65923..07f59d48 100644
--- a/deploy/crds/awxrestore_v1beta1_crd.yaml
+++ b/deploy/crds/awxrestore_v1beta1_crd.yaml
@@ -21,7 +21,7 @@ spec:
         openAPIV3Schema:
           type: object
           x-kubernetes-preserve-unknown-fields: true
-          description: Schema validation for the AWXBackup CRD
+          description: Schema validation for the AWXRestore CRD
           properties:
             spec:
               type: object
@@ -30,10 +30,10 @@ spec:
                   description: Name of the deployment to be restored to
                   type: string
                 tower_backup_pvc:
-                  description: Name of the PVC to be used for storing the backup
+                  description: Name of the PVC to be restored from
                   type: string
                 tower_backup_pvc_namespace:
-                  description: Namespace PVC is in
+                  description: Namespace the PVC is in
                   type: string
                 tower_backup_dir:
                   description: Backup directory name, a status found on the awxbackup object (towerBackupComplete)
@@ -51,4 +51,4 @@ spec:
                   description: Custom postgres_configuration secret name
                   type: string
               oneOf:
-                - required: ["tower_name", "tower_backup_pvc"]
+                - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"]
diff --git a/roles/backup/README.md b/roles/backup/README.md
index 3d93035c..46215b85 100644
--- a/roles/backup/README.md
+++ b/roles/backup/README.md
@@ -1,12 +1,11 @@
-Role Name
+Backup Role
 =========
 
-The purpose of this role is to create a backup of your AWX deployment.  This includes:
+The purpose of this role is to create a backup of your AWX deployment which includes:
+  - custom deployment specific values in the spec section of the AWX custom resource object
   - backup of the postgresql database
-  - secret_key
-  - custom user config files
-  - manual projects
-
+  - secret_key, admin_password, and broadcast_websocket secrets
+  - database configuration
 
 Requirements
 ------------
@@ -32,13 +31,12 @@ spec:
   tower_name: mytower
 ```
 
-Note that the `tower_name` above is the name of the AWX deployment you intend to backup from.  
-
+Note that the `tower_name` above is the name of the AWX deployment you intend to backup from.  The namespace above is the one containing the AWX deployment that will be backed up.
 
 Finally, use `kubectl` to create the backup object in your cluster:
 
 ```bash
-#> kubectl apply -f backup-awx.yml
+$ kubectl apply -f backup-awx.yml
 ```
 
 The resulting pvc will contain a backup tar that can be used to restore to a new deployment. Future backups will also be stored in separate tars on the same pvc.
@@ -62,6 +60,14 @@ tower_backup_storage_class: 'standard'
 tower_backup_size: '20Gi'
 ```
 
+By default, the backup pvc will be created in the `default` namespace.  If you want your backup to be stored
+in a specific namespace, you can do so by specifying `tower_backup_pvc_namespace`.  Keep in mind that you will
+need to provide the same namespace when restoring.  
+
+```
+tower_backup_pvc_namespace: 'custom-namespace'
+```
+
 If a custom postgres configuration secret was used when deploying AWX, it must be set:
 
 ```
diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml
index c48a9cf2..3d3cd361 100644
--- a/roles/backup/meta/main.yml
+++ b/roles/backup/meta/main.yml
@@ -18,6 +18,7 @@ galaxy_info:
 
   galaxy_tags:
     - tower
+    - controller
     - awx
     - ansible
     - backup
diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml
index 119c68e1..69699796 100644
--- a/roles/backup/tasks/init.yml
+++ b/roles/backup/tasks/init.yml
@@ -40,7 +40,7 @@
   set_fact:
     _default_backup_pvc: "{{ meta.name }}-backup-claim"
 
-# by default, it will re-use the old pvc if already created (unless pvc is provided)
+# by default, it will re-use the old pvc if already created (unless a pvc is provided)
 - name: Set PVC to use for backup
   set_fact:
     backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}"
diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml
index e682e860..84f3ead3 100644
--- a/roles/backup/tasks/main.yml
+++ b/roles/backup/tasks/main.yml
@@ -34,4 +34,4 @@
 - name: Update status variables
   include_tasks: update_status.yml
 
-# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object.  See ticket
+# TODO: backup tower settings or make sure that users only specify settings/config changes via AWX object.  See ticket
diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml
index 07ed178e..e9ca54ce 100644
--- a/roles/backup/tasks/postgres.yml
+++ b/roles/backup/tasks/postgres.yml
@@ -34,7 +34,9 @@
     label_selectors:
       - "app.kubernetes.io/name={{ tower_name }}-postgres"
   register: postgres_pod
-  until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
   delay: 5
   retries: 60
 
@@ -71,7 +73,7 @@
     command: >-
       bash -c "chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db"
 
-- name: Get the postgres pod information
+- name: Set full resolvable host name for postgres pod
   set_fact:
     resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local"
 
diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml
index a250ddf7..6ef987b4 100644
--- a/roles/backup/tasks/secrets.yml
+++ b/roles/backup/tasks/secrets.yml
@@ -1,11 +1,5 @@
 ---
 
-- name: Make temp secrets directory
-  file:
-    path: "/tmp/secrets" #-{{ lookup('password', '/dev/null chars=ascii_lowercase,digits length=8')}}"
-    state: directory
-  register: secrets_dir
-
 - name: Get secret_key
   k8s_info:
     kind: Secret
@@ -42,7 +36,7 @@
 - name: Get postgres configuration
   k8s_info:
     kind: Secret
-    namespace: '{{ tower_backup_pvc_namespace }}'
+    namespace: '{{ meta.namespace }}'
     name: '{{ tower_postgres_configuration_secret }}'
   register: _postgres_configuration
 
@@ -57,7 +51,7 @@
 
 - name: Template secrets into yaml
   set_fact:
-    secrets_file: "{{ lookup('template', 'secrets.yml.j2')}}"
+    secrets_file: "{{ lookup('template', 'secrets.yml.j2') }}"
 
 - name: Write postgres configuration to pvc
   k8s_exec:
diff --git a/roles/backup/templates/admin_password_secret.yml.j2 b/roles/backup/templates/admin_password_secret.yml.j2
deleted file mode 100644
index 8adc8177..00000000
--- a/roles/backup/templates/admin_password_secret.yml.j2
+++ /dev/null
@@ -1,10 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-{% raw %}
-  name: '{{ tower_name }}'
-  namespace: '{{ meta.namespace }}'
-{% endraw %}
-stringData:
-  password: '{{ admin_password }}'
diff --git a/roles/backup/templates/broadcast_websocket_secret.yml.j2 b/roles/backup/templates/broadcast_websocket_secret.yml.j2
deleted file mode 100644
index e590ab06..00000000
--- a/roles/backup/templates/broadcast_websocket_secret.yml.j2
+++ /dev/null
@@ -1,10 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-{% raw %}
-  name: '{{ tower_name }}-broadcast-websocket'
-  namespace: '{{ meta.namespace }}'
-{% endraw %}
-stringData:
-  secret: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}'
diff --git a/roles/backup/templates/postgres_secret.yml.j2 b/roles/backup/templates/postgres_secret.yml.j2
deleted file mode 100644
index f5fa75d1..00000000
--- a/roles/backup/templates/postgres_secret.yml.j2
+++ /dev/null
@@ -1,16 +0,0 @@
-# Postgres Secret.
----
-apiVersion: v1
-kind: Secret
-metadata:
-{% raw %}
-  name: '{{ tower_name }}-postgres-configuration'
-  namespace: '{{ meta.namespace }}'
-{% endraw %}
-stringData:
-  password: '{{ database_password }}'
-  username: '{{ database_username }}'
-  database: '{{ database_name }}'
-  port: '{{ database_port }}'
-  host: '{{ database_host }}'
-  type: '{{ database_type }}'
diff --git a/roles/backup/templates/secret_key_secret.yml.j2 b/roles/backup/templates/secret_key_secret.yml.j2
deleted file mode 100644
index ae60b37f..00000000
--- a/roles/backup/templates/secret_key_secret.yml.j2
+++ /dev/null
@@ -1,10 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-{% raw %}
-  name: '{{ tower_name }}'
-  namespace: '{{ meta.namespace }}'
-{% endraw %}
-stringData:
-  secret_key: '{{ secret_key }}'
diff --git a/roles/restore/README.md b/roles/restore/README.md
index ae262c11..0d9eb1fb 100644
--- a/roles/restore/README.md
+++ b/roles/restore/README.md
@@ -1,10 +1,12 @@
-Role Name
+Restore Role
 =========
 
-The purpose of this role is to restore your AWX deployment from an existing PVC backup. The backup should include:
+The purpose of this role is to restore your AWX deployment from an existing PVC backup. The backup includes:
+  - custom deployment specific values in the spec section of the AWX custom resource object
   - backup of the postgresql database
-  - secrets, included the secret_key.  
-  - AWX custom resource object with deployment specific settings
+  - secret_key, admin_password, and broadcast_websocket secrets
+  - database configuration
+
 
 
 Requirements
@@ -44,7 +46,7 @@ kubectl create ns my-namespace
 Finally, use `kubectl` to create the restore object in your cluster:
 
 ```bash
-#> kubectl apply -f restore-awx.yml
+$ kubectl apply -f restore-awx.yml
 ```
 
 This will create a new deployment and restore your backup to it.  
diff --git a/roles/restore/meta/main.yml b/roles/restore/meta/main.yml
index f9e95b3b..720b9bcb 100644
--- a/roles/restore/meta/main.yml
+++ b/roles/restore/meta/main.yml
@@ -18,6 +18,7 @@ galaxy_info:
 
   galaxy_tags:
     - tower
+    - controller
     - awx
     - ansible
     - restore
diff --git a/roles/restore/tasks/apply_secret.yml b/roles/restore/tasks/apply_secret.yml
deleted file mode 100644
index a71004cc..00000000
--- a/roles/restore/tasks/apply_secret.yml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-
-- name: Get secret definition from pvc
-  k8s_exec:
-    namespace: "{{ tower_backup_pvc_namespace }}"
-    pod: "{{ meta.name }}-db-management"
-    command: >-
-      bash -c "cat '{{ tower_backup_dir }}/{{ item }}.yml'"
-  register: awx_object
-
-- name: Write temp secret definition template file
-  copy:
-    dest: "{{ definitions_dir.path }}/{{ item }}.yml.j2"
-    content: |
-      {{ awx_object.stdout }}
-    mode: '0600'
-
-- name: Apply secret
-  k8s:
-    state: "{{ state | default('present') }}"
-    namespace: "{{ namespace | default('default') }}"
-    apply: yes
-    wait: yes
-    template: "{{ definitions_dir.path }}/{{ item }}.yml.j2"
diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/deploy_awx.yml
similarity index 74%
rename from roles/restore/tasks/init_awx.yml
rename to roles/restore/tasks/deploy_awx.yml
index eb8c3bf9..4332c5e3 100644
--- a/roles/restore/tasks/init_awx.yml
+++ b/roles/restore/tasks/deploy_awx.yml
@@ -8,7 +8,8 @@
       bash -c "cat '{{ tower_backup_dir }}/awx_object'"
   register: awx_object
 
-- set_fact:
+- name: Set AWX spec variable from backup
+  set_fact:
     awx_spec: "{{ awx_object.stdout }}"
 
 - name: Deploy AWX
@@ -16,11 +17,11 @@
     state: "{{ state | default('present') }}"
     namespace: "{{ meta.namespace | default('default') }}"
     apply: yes
-    wait: yes
     template: awx_object.yml.j2
-
-# TODO: The awx object and secrets need to be applied from the awx-operator, because that is where the service account is?
-#       So we will need to either copy them over or pipe them into a template command
+    wait: true
+    wait_condition:
+      type: "Running"
+      status: "True"
 
 # TODO: Add logic to allow users to provide override values here,
 #       or to specify spec values that were not in the backed up AWX object.
diff --git a/roles/restore/tasks/init_secrets.yml b/roles/restore/tasks/init_secrets.yml
deleted file mode 100644
index 636c4f68..00000000
--- a/roles/restore/tasks/init_secrets.yml
+++ /dev/null
@@ -1,19 +0,0 @@
----
-
-- name: Get secret definition from pvc
-  k8s_exec:
-    namespace: "{{ tower_backup_pvc_namespace }}"
-    pod: "{{ meta.name }}-db-management"
-    command: >-
-      bash -c "cat '{{ tower_backup_dir }}/secrets.yml'"
-  register: secrets
-
-- include_vars: "{{ secrets.stdout | from_yaml }}"
-
-- name: Apply secret
-  k8s:
-    state: present
-    namespace: "{{ meta.namespace | default('default') }}"
-    apply: yes
-    wait: yes
-    template: "secrets.yml.j2"
diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml
index 63d1138d..48461ba3 100644
--- a/roles/restore/tasks/main.yml
+++ b/roles/restore/tasks/main.yml
@@ -16,12 +16,12 @@
 - block:
     - include_tasks: init.yml
 
-    - include_tasks: init_awx.yml
+    - include_tasks: secrets.yml
+
+    - include_tasks: deploy_awx.yml
 
     - include_tasks: postgres.yml
 
-    - include_tasks: secrets.yml
-
     - name: Set flag signifying this restore was successful
       set_fact:
         tower_restore_complete: True
diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml
index 75cdc62d..13e38816 100644
--- a/roles/restore/tasks/postgres.yml
+++ b/roles/restore/tasks/postgres.yml
@@ -34,7 +34,9 @@
     label_selectors:
       - "app.kubernetes.io/name={{ tower_name }}-postgres"
   register: postgres_pod
-  until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
   delay: 5
   retries: 60
 
@@ -59,7 +61,7 @@
     replicas: 0
   when: this_deployment['resources'] | length
 
-- name: Get the postgres pod information
+- name: Set full resolvable host name for postgres pod
   set_fact:
     resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local"
 
diff --git a/roles/restore/tasks/secrets.yml b/roles/restore/tasks/secrets.yml
new file mode 100644
index 00000000..1a9cd864
--- /dev/null
+++ b/roles/restore/tasks/secrets.yml
@@ -0,0 +1,37 @@
+---
+
+- name: Get secret definition from pvc
+  k8s_exec:
+    namespace: "{{ tower_backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "cat '{{ tower_backup_dir }}/secrets.yml'"
+  register: secrets
+
+- name: Create temp vars file
+  tempfile:
+    prefix: secret_vars-
+  register: secret_vars
+
+- name: Write vars to file locally
+  copy:
+    dest: "{{ secret_vars.path }}"
+    content: "{{ secrets.stdout }}"
+    mode: 0640
+
+- name: Include secret vars from backup
+  include_vars: "{{ secret_vars.path }}"
+
+- name: Set new database host based on supplied tower_name
+  set_fact:
+    database_host: "{{ tower_name }}-postgres"
+  when:
+    - database_type == 'managed'
+
+- name: Apply secret
+  k8s:
+    state: present
+    namespace: "{{ meta.namespace | default('default') }}"
+    apply: yes
+    wait: yes
+    template: "secrets.yml.j2"
diff --git a/roles/restore/templates/awx_object.yml.j2 b/roles/restore/templates/awx_object.yml.j2
index e9638b48..b13e3a51 100644
--- a/roles/restore/templates/awx_object.yml.j2
+++ b/roles/restore/templates/awx_object.yml.j2
@@ -1,5 +1,5 @@
 ---
-apiVersion: '{{ awx_api_version }}'
+apiVersion: '{{ api_version }}'
 kind: AWX
 metadata:
   name: '{{ tower_name }}'
diff --git a/roles/restore/templates/secrets.yml.j2 b/roles/restore/templates/secrets.yml.j2
index a4fee4fe..019af5fa 100644
--- a/roles/restore/templates/secrets.yml.j2
+++ b/roles/restore/templates/secrets.yml.j2
@@ -18,10 +18,8 @@ stringData:
 apiVersion: v1
 kind: Secret
 metadata:
-{% raw %}
   name: '{{ tower_name }}'
   namespace: '{{ meta.namespace }}'
-{% endraw %}
 stringData:
   secret_key: '{{ secret_key }}'
 
@@ -30,10 +28,8 @@ stringData:
 apiVersion: v1
 kind: Secret
 metadata:
-{% raw %}
   name: '{{ tower_name }}'
   namespace: '{{ meta.namespace }}'
-{% endraw %}
 stringData:
   password: '{{ admin_password }}'
 
@@ -42,9 +38,7 @@ stringData:
 apiVersion: v1
 kind: Secret
 metadata:
-{% raw %}
   name: '{{ tower_name }}-broadcast-websocket'
   namespace: '{{ meta.namespace }}'
-{% endraw %}
 stringData:
   secret: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}'