Merge pull request #133 from rooftopcellist/backup-role

Backup role for awx-operator

README.md

@@ -229,9 +229,12 @@ stringData:
   username: <username to connect as>
   password: <password to connect with>
   sslmode: prefer
+  type: unmanaged
 type: Opaque
 ```
 
+> It is possible to set a specific username, password, port, or database, but still have the database managed by the operator. In this case, when creating the postgres-configuration secret, the `type: managed` field should be added.  
+
 **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`.
 
 #### Migrating data from an old AWX instance
@@ -665,7 +668,7 @@ After it is built, test it on a local cluster:
 #> minikube addons enable ingress
 #> ansible-playbook ansible/deploy-operator.yml -e operator_image=quay.io/<user>/awx-operator -e operator_version=test
 #> kubectl create namespace example-awx
-#> ansible-playbook ansible/instantiate-awx-deployment.yml -e tower_namespace=example-awx
+#> ansible-playbook ansible/instantiate-awx-deployment.yml -e namespace=example-awx
 #> <test everything>
 #> minikube delete
 ```

ansible/chain-operator-files.yml

@@ -6,12 +6,24 @@
   gather_facts: false
 
   tasks:
-    - name: Template CRD
+    - name: Template AWX CRD
       template:
         src: crd.yml.j2
         dest: "{{ playbook_dir }}/../deploy/crds/awx_v1beta1_crd.yaml"
         mode: '0644'
 
+    - name: Template AWXBackup CRD
+      template:
+        src: awxbackup_crd.yml.j2
+        dest: "{{ playbook_dir }}/../deploy/crds/awxbackup_v1beta1_crd.yaml"
+        mode: '0644'
+
+    - name: Template AWXRestore CRD
+      template:
+        src: awxrestore_crd.yml.j2
+        dest: "{{ playbook_dir }}/../deploy/crds/awxrestore_v1beta1_crd.yaml"
+        mode: '0644'
+
     - name: Template awx-operator.yaml
       template:
         src: awx-operator.yaml.j2

ansible/instantiate-awx-deployment.yml

@@ -9,7 +9,7 @@
     - name: Deploy AWX
       k8s:
         state: "{{ state | default('present') }}"
-        namespace: "{{ tower_namespace | default('default') }}"
+        namespace: "{{ namespace | default('default') }}"
         apply: yes
         wait: yes
         definition:

ansible/templates/awx-operator.yaml.j2

@@ -3,6 +3,10 @@
 # Update templates under ansible/templates/
 {% include 'crd.yml.j2' %}
 
+{% include 'awxbackup_crd.yml.j2' %}
+
+{% include 'awxrestore_crd.yml.j2' %}
+
 {% include 'role.yml.j2' %}
 
 {% include 'role_binding.yml.j2' %}

ansible/templates/awxbackup_crd.yml.j2

@@ -0,0 +1,61 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxbackups.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXBackup
+    listKind: AWXBackupList
+    plural: awxbackups
+    singular: awxbackup
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the AWXBackup CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                deployment_name:
+                  description: Name of the deployment to be backed up
+                  type: string
+                backup_pvc:
+                  description: Name of the PVC to be used for storing the backup
+                  type: string
+                backup_pvc_namespace:
+                  description: Namespace PVC is in
+                  type: string
+                backup_storage_requirements:
+                  description: Storage requirements for the PostgreSQL container
+                  type: string
+                backup_storage_class:
+                  description: Storage class to use when creating PVC for backup
+                  type: string
+                tower_secret_key_secret:
+                  description: Custom secret_key secret name
+                  type: string
+                tower_admin_password_secret:
+                  description: Custom admin_password secret name
+                  type: string
+                tower_broadcast_websocket_secret:
+                  description: Custom broadcast_websocket secret name
+                  type: string
+                tower_postgres_configuration_secret:
+                  description: Custom postgres_configuration secret name
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for backing up data
+                  type: string
+
+              oneOf:
+                - required: ["deployment_name"]

ansible/templates/awxrestore_crd.yml.j2

@@ -0,0 +1,60 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxrestores.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXRestore
+    listKind: AWXRestoreList
+    plural: awxrestores
+    singular: awxrestore
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the AWXRestore CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                deployment_name:
+                  description: Name of the deployment to be restored to
+                  type: string
+                backup:
+                  description: AWXBackup object name
+                  type: string
+                backup_pvc:
+                  description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim)
+                  type: string
+                backup_pvc_namespace:
+                  description: Namespace the PVC is in
+                  type: string
+                backup_dir:
+                  description: Backup directory name, set as a status found on the awxbackup object (backupDirectory)
+                  type: string
+                tower_secret_key_secret:
+                  description: Custom secret_key secret name
+                  type: string
+                tower_admin_password_secret:
+                  description: Custom admin_password secret name
+                  type: string
+                tower_broadcast_websocket_secret:
+                  description: Custom broadcast_websocket secret name
+                  type: string
+                tower_postgres_configuration_secret:
+                  description: Custom postgres_configuration secret name
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for backing up data
+                  type: string
+              oneOf:
+                - required: ["deployment_name", "backup_pvc_namespace"]

ansible/templates/crd.yml.j2

@@ -58,6 +58,9 @@ spec:
                 tower_old_postgres_configuration_secret:
                   description: Secret where the old database configuration can be found for data migration
                   type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for data migration
+                  type: string
                 tower_secret_key_secret:
                   description: Secret where the secret key can be found
                   type: string
@@ -346,7 +349,16 @@ spec:
                   description: Admin user of the deployed instance
                   type: string
                 towerAdminPasswordSecret:
-                  description: Admin password of the deployed instance
+                  description: Admin password secret name of the deployed instance
+                  type: string
+                towerPostgresConfigurationSecret:
+                  description: Postgres Configuration secret name of the deployed instance
+                  type: string
+                towerBroadcastWebsocketSecret:
+                  description: Broadcast websocket secret name of the deployed instance
+                  type: string
+                towerSecretKeySecret:
+                  description: Secret key secret name of the deployed instance
                   type: string
                 towerMigratedFromSecret:
                   description: The secret used for migrating an old Tower.

ansible/templates/role.yml.j2

@@ -79,5 +79,7 @@ rules:
       - awx.ansible.com
     resources:
       - '*'
+      - awxbackups
+      - awxrestores
     verbs:
       - '*'

deploy/awx-operator.yaml

@@ -60,6 +60,9 @@ spec:
                 tower_old_postgres_configuration_secret:
                   description: Secret where the old database configuration can be found for data migration
                   type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for data migration
+                  type: string
                 tower_secret_key_secret:
                   description: Secret where the secret key can be found
                   type: string
@@ -348,7 +351,16 @@ spec:
                   description: Admin user of the deployed instance
                   type: string
                 towerAdminPasswordSecret:
-                  description: Admin password of the deployed instance
+                  description: Admin password secret name of the deployed instance
+                  type: string
+                towerPostgresConfigurationSecret:
+                  description: Postgres Configuration secret name of the deployed instance
+                  type: string
+                towerBroadcastWebsocketSecret:
+                  description: Broadcast websocket secret name of the deployed instance
+                  type: string
+                towerSecretKeySecret:
+                  description: Secret key secret name of the deployed instance
                   type: string
                 towerMigratedFromSecret:
                   description: The secret used for migrating an old Tower.
@@ -376,6 +388,129 @@ spec:
               type: object
           type: object
 
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxbackups.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXBackup
+    listKind: AWXBackupList
+    plural: awxbackups
+    singular: awxbackup
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the AWXBackup CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                deployment_name:
+                  description: Name of the deployment to be backed up
+                  type: string
+                backup_pvc:
+                  description: Name of the PVC to be used for storing the backup
+                  type: string
+                backup_pvc_namespace:
+                  description: Namespace PVC is in
+                  type: string
+                backup_storage_requirements:
+                  description: Storage requirements for the PostgreSQL container
+                  type: string
+                backup_storage_class:
+                  description: Storage class to use when creating PVC for backup
+                  type: string
+                tower_secret_key_secret:
+                  description: Custom secret_key secret name
+                  type: string
+                tower_admin_password_secret:
+                  description: Custom admin_password secret name
+                  type: string
+                tower_broadcast_websocket_secret:
+                  description: Custom broadcast_websocket secret name
+                  type: string
+                tower_postgres_configuration_secret:
+                  description: Custom postgres_configuration secret name
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for backing up data
+                  type: string
+
+              oneOf:
+                - required: ["deployment_name"]
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxrestores.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXRestore
+    listKind: AWXRestoreList
+    plural: awxrestores
+    singular: awxrestore
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the AWXRestore CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                deployment_name:
+                  description: Name of the deployment to be restored to
+                  type: string
+                backup:
+                  description: AWXBackup object name
+                  type: string
+                backup_pvc:
+                  description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim)
+                  type: string
+                backup_pvc_namespace:
+                  description: Namespace the PVC is in
+                  type: string
+                backup_dir:
+                  description: Backup directory name, set as a status found on the awxbackup object (backupDirectory)
+                  type: string
+                tower_secret_key_secret:
+                  description: Custom secret_key secret name
+                  type: string
+                tower_admin_password_secret:
+                  description: Custom admin_password secret name
+                  type: string
+                tower_broadcast_websocket_secret:
+                  description: Custom broadcast_websocket secret name
+                  type: string
+                tower_postgres_configuration_secret:
+                  description: Custom postgres_configuration secret name
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for backing up data
+                  type: string
+              oneOf:
+                - required: ["deployment_name", "backup_pvc_namespace"]
+
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -457,6 +592,8 @@ rules:
       - awx.ansible.com
     resources:
       - '*'
+      - awxbackups
+      - awxrestores
     verbs:
       - '*'
 

deploy/crds/awx_v1beta1_crd.yaml

@@ -58,6 +58,9 @@ spec:
                 tower_old_postgres_configuration_secret:
                   description: Secret where the old database configuration can be found for data migration
                   type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for data migration
+                  type: string
                 tower_secret_key_secret:
                   description: Secret where the secret key can be found
                   type: string
@@ -346,7 +349,16 @@ spec:
                   description: Admin user of the deployed instance
                   type: string
                 towerAdminPasswordSecret:
-                  description: Admin password of the deployed instance
+                  description: Admin password secret name of the deployed instance
+                  type: string
+                towerPostgresConfigurationSecret:
+                  description: Postgres Configuration secret name of the deployed instance
+                  type: string
+                towerBroadcastWebsocketSecret:
+                  description: Broadcast websocket secret name of the deployed instance
+                  type: string
+                towerSecretKeySecret:
+                  description: Secret key secret name of the deployed instance
                   type: string
                 towerMigratedFromSecret:
                   description: The secret used for migrating an old Tower.

deploy/crds/awxbackup_v1beta1_crd.yaml

@@ -0,0 +1,61 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxbackups.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXBackup
+    listKind: AWXBackupList
+    plural: awxbackups
+    singular: awxbackup
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the AWXBackup CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                deployment_name:
+                  description: Name of the deployment to be backed up
+                  type: string
+                backup_pvc:
+                  description: Name of the PVC to be used for storing the backup
+                  type: string
+                backup_pvc_namespace:
+                  description: Namespace PVC is in
+                  type: string
+                backup_storage_requirements:
+                  description: Storage requirements for the PostgreSQL container
+                  type: string
+                backup_storage_class:
+                  description: Storage class to use when creating PVC for backup
+                  type: string
+                tower_secret_key_secret:
+                  description: Custom secret_key secret name
+                  type: string
+                tower_admin_password_secret:
+                  description: Custom admin_password secret name
+                  type: string
+                tower_broadcast_websocket_secret:
+                  description: Custom broadcast_websocket secret name
+                  type: string
+                tower_postgres_configuration_secret:
+                  description: Custom postgres_configuration secret name
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for backing up data
+                  type: string
+
+              oneOf:
+                - required: ["deployment_name"]

deploy/crds/awxrestore_v1beta1_crd.yaml

@@ -0,0 +1,60 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxrestores.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXRestore
+    listKind: AWXRestoreList
+    plural: awxrestores
+    singular: awxrestore
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the AWXRestore CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                deployment_name:
+                  description: Name of the deployment to be restored to
+                  type: string
+                backup:
+                  description: AWXBackup object name
+                  type: string
+                backup_pvc:
+                  description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim)
+                  type: string
+                backup_pvc_namespace:
+                  description: Namespace the PVC is in
+                  type: string
+                backup_dir:
+                  description: Backup directory name, set as a status found on the awxbackup object (backupDirectory)
+                  type: string
+                tower_secret_key_secret:
+                  description: Custom secret_key secret name
+                  type: string
+                tower_admin_password_secret:
+                  description: Custom admin_password secret name
+                  type: string
+                tower_broadcast_websocket_secret:
+                  description: Custom broadcast_websocket secret name
+                  type: string
+                tower_postgres_configuration_secret:
+                  description: Custom postgres_configuration secret name
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for backing up data
+                  type: string
+              oneOf:
+                - required: ["deployment_name", "backup_pvc_namespace"]

docs/migration.md

@@ -6,14 +6,14 @@ To migrate data from an older AWX installation, you must provide some informatio
 
 ### Secret Key
 
-You can find your old secret key in the inventory file you used to deploy AWX in releases prior to version 18. 
+You can find your old secret key in the inventory file you used to deploy AWX in releases prior to version 18.
 
 ```yaml
 apiVersion: v1
 kind: Secret
 metadata:
   name: <resourcename>-secret-key
-  namespace: <target namespace>
+  namespace: <target-namespace>
 stringData:
   secret_key: <old-secret-key>
 type: Opaque
@@ -49,6 +49,9 @@ In the next section pass it in through `tower_postgres_configuration_secret` ins
 from the key and ensuring the value matches the name of the secret. This will make AWX pick up on the existing
 database and apply any pending migrations. It is strongly recommended to backup your database beforehand.
 
+The postgresql pod for the old deployment is used when streaming data to the new postgresql pod.  If your postgresql pod has a custom label,
+you can pass that via the `postgres_label_selector` variable to make sure the postgresql pod can be found.
+
 ## Deploy AWX
 
 When you apply your AWX object, you must specify the name to the database secret you created above:

molecule/default/prepare.yml

@@ -11,10 +11,18 @@
     - "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/ansible/group_vars/all"
 
   tasks:
-    - name: Create Custom Resource Definition
+    - name: Create AWX Custom Resource Definition
       k8s:
         definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_crd.yaml'])) }}"
 
+    - name: Create AWXBackup Custom Resource Definition
+      k8s:
+        definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxbackup_v1beta1_crd.yaml'])) }}"
+
+    - name: Create AWXRestore Custom Resource Definition
+      k8s:
+        definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxrestore_v1beta1_crd.yaml'])) }}"
+
     - name: Ensure specified namespace is present
       k8s:
         api_version: v1

molecule/test-local/converge.yml

@@ -29,10 +29,9 @@
     operator_image: awx.ansible.com/awx-operator
     operator_version: testing
     custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_molecule.yaml'])) | from_yaml }}"
+
   tasks:
-
     - block:
-
         - name: Delete the Operator Deployment
           k8s:
             state: absent

roles/backup/README.md

@@ -0,0 +1,95 @@
+Backup Role
+=========
+
+The purpose of this role is to create a backup of your AWX deployment which includes:
+  - custom deployment specific values in the spec section of the AWX custom resource object
+  - backup of the postgresql database
+  - secret_key, admin_password, and broadcast_websocket secrets
+  - database configuration
+
+Requirements
+------------
+
+This role assumes you are authenticated with an Openshift or Kubernetes cluster:
+  - The awx-operator has been deployed to the cluster
+  - AWX is deployed to via the operator
+
+
+Usage
+----------------
+
+Then create a file named `backup-awx.yml` with the following contents:
+
+```yaml
+---
+apiVersion: awx.ansible.com/v1beta1
+kind: AWXBackup
+metadata:
+  name: awxbackup-2021-04-22
+  namespace: my-namespace
+spec:
+  deployment_name: mytower
+```
+
+Note that the `deployment_name` above is the name of the AWX deployment you intend to backup from.  The namespace above is the one containing the AWX deployment that will be backed up.
+
+Finally, use `kubectl` to create the backup object in your cluster:
+
+```bash
+$ kubectl apply -f backup-awx.yml
+```
+
+The resulting pvc will contain a backup tar that can be used to restore to a new deployment. Future backups will also be stored in separate tars on the same pvc.
+
+
+Role Variables
+--------------
+
+A custom, pre-created pvc can be used by setting the following variables.  
+
+```
+backup_pvc: 'awx-backup-volume-claim'
+```
+
+> If no pvc or storage class is provided, the cluster's default storage class will be used to create the pvc.
+
+This role will automatically create a pvc using a Storage Class if provided:
+
+```
+backup_storage_class: 'standard'
+backup_storage_requirements: '20Gi'
+```
+
+By default, the backup pvc will be created in the same namespace the awxbackup object is created in.  If you want your backup to be stored
+in a specific namespace, you can do so by specifying `backup_pvc_namespace`.  Keep in mind that you will
+need to provide the same namespace when restoring.  
+
+```
+backup_pvc_namespace: 'custom-namespace'
+```
+
+If a custom postgres configuration secret was used when deploying AWX, it will automatically be used by the backup role.  
+To check the name of this secret, look at the towerPostgresConfigurationSecret status on your AWX object.  
+
+The postgresql pod for the old deployment is used when backing up data to the new postgresql pod.  If your postgresql pod has a custom label,
+you can pass that via the `postgres_label_selector` variable to make sure the postgresql pod can be found.
+
+
+Testing
+----------------
+
+You can test this role directly by creating and running the following playbook with the appropriate variables:
+
+```
+---
+- name: Backup AWX
+  hosts: localhost
+  gather_facts: false
+  roles:
+    - backup
+```
+
+License
+-------
+
+MIT

roles/backup/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+# Required: specify name of tower deployment to backup from
+deployment_name: ''
+kind: 'AWXBackup'
+api_version: '{{ deployment_type }}.ansible.com/v1beta1'
+
+# Specify a pre-created PVC (name) to backup to
+backup_pvc: ''
+backup_pvc_namespace: "{{ meta.namespace }}"
+
+# Size of backup PVC if created dynamically
+backup_storage_requirements: ''
+
+# Specify storage class to determine how to dynamically create PVC's with
+backup_storage_class: ''

roles/backup/meta/main.yml

@@ -0,0 +1,31 @@
+---
+galaxy_info:
+  author: Ansible
+  description: AWX role for AWX Operator for Kubernetes.
+  company: Red Hat, Inc.
+
+  license: MIT
+
+  min_ansible_version: 2.8
+
+  platforms:
+    - name: EL
+      versions:
+        - all
+    - name: Debian
+      versions:
+        - all
+
+  galaxy_tags:
+    - tower
+    - controller
+    - awx
+    - ansible
+    - backup
+    - automation
+
+dependencies: []
+
+collections:
+  - community.kubernetes
+  - operator_sdk.util

roles/backup/tasks/awx-cro.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Get AWX custom resource object
+  k8s_info:
+    version: v1beta1
+    kind: AWX
+    namespace: '{{ meta.namespace }}'
+    name: '{{ deployment_name }}'
+  register: _awx_cro
+
+- name: Set AWX object
+  set_fact:
+    _awx: "{{ _awx_cro['resources'][0] }}"
+
+- name: Set user specified spec
+  set_fact:
+    awx_spec: "{{ _awx['spec'] }}"
+
+- name: Write awx object to pvc
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "echo '{{ awx_spec }}' > {{ backup_dir }}/awx_object"

roles/backup/tasks/cleanup.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Delete any existing management pod
+  k8s:
+    name: "{{ meta.name }}-db-management"
+    kind: Pod
+    namespace: "{{ backup_pvc_namespace }}"
+    state: absent
+    force: true

roles/backup/tasks/error_handling.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Determine the timestamp
+  set_fact:
+    now: '{{ lookup("pipe", "date +%FT%TZ") }}'
+
+- name: Emit ocp event with error
+  k8s:
+    kind: Event
+    namespace: "{{ meta.namespace }}"
+    template: "event.yml.j2"

roles/backup/tasks/init.yml

@@ -0,0 +1,69 @@
+---
+
+- name: Delete any existing management pod
+  k8s:
+    name: "{{ meta.name }}-db-management"
+    kind: Pod
+    namespace: "{{ backup_pvc_namespace }}"
+    state: absent
+    force: true
+    wait: true
+
+# Check to make sure provided pvc exists, error loudly if not.  Otherwise, the management pod will just stay in pending state forever.
+- name: Check provided PVC exists
+  k8s_info:
+    name: "{{ backup_pvc }}"
+    kind: PersistentVolumeClaim
+    namespace: "{{ backup_pvc_namespace }}"
+  register: provided_pvc
+  when:
+    - backup_pvc != ''
+
+- name: Surface error to user
+  block:
+    - name: Set error message
+      set_fact:
+        error_msg: "{{ backup_pvc }} does not exist, please create this pvc first."
+
+    - name: Handle error
+      import_tasks: error_handling.yml
+
+    - name: Fail early if pvc is defined but does not exist
+      fail:
+        msg: "{{ backup_pvc }} does not exist, please create this pvc first."
+  when:
+    - backup_pvc != ''
+    - provided_pvc.resources | length == 0
+
+# If backup_pvc is defined, use in management-pod.yml.j2
+- name: Set default pvc name
+  set_fact:
+    _default_backup_pvc: "{{ deployment_name }}-backup-claim"
+
+# by default, it will re-use the old pvc if already created (unless a pvc is provided)
+- name: Set PVC to use for backup
+  set_fact:
+    backup_claim: "{{ backup_pvc | default(_default_backup_pvc, true) }}"
+
+- name: Create PVC for backup
+  k8s:
+    kind: PersistentVolumeClaim
+    template: "backup_pvc.yml.j2"
+  when:
+    - backup_pvc == '' or backup_pvc is not defined
+
+- name: Create management pod from templated deployment config
+  k8s:
+    name: "{{ meta.name }}-db-management"
+    kind: Deployment
+    state: present
+    template: "management-pod.yml.j2"
+    wait: true
+
+- name: Look up details for this deployment
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "AWX"
+    name: "{{ deployment_name }}"
+    namespace: "{{ meta.namespace }}"
+  register: this_awx

roles/backup/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+
+- name: Look up details for this backup object
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+  register: this_backup
+
+- block:
+    - include_tasks: init.yml
+
+    - include_tasks: postgres.yml
+
+    - include_tasks: secrets.yml
+
+    - include_tasks: awx-cro.yml
+
+    - name: Set flag signifying this backup was successful
+      set_fact:
+        backup_complete: true
+
+    - include_tasks: cleanup.yml
+
+  when:
+    - this_backup['resources'][0]['status']['backupDirectory'] is not defined
+
+- name: Update status variables
+  include_tasks: update_status.yml
+
+# TODO: backup tower settings or make sure that users only specify settings/config changes via AWX object.  See ticket

roles/backup/tasks/postgres.yml

@@ -0,0 +1,95 @@
+---
+
+- name: Get PostgreSQL configuration
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: "{{ this_awx['resources'][0]['status']['towerPostgresConfigurationSecret'] }}"
+  register: pg_config
+
+- name: Fail if postgres configuration secret status does not exist
+  fail:
+    msg: "The towerPostgresConfigurationSecret status is not set on the AWX object yet or the secret has been deleted."
+  when: not pg_config | default([]) | length
+
+- name: Store Database Configuration
+  set_fact:
+    awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
+    awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}"
+    awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
+    awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
+    awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
+    awx_postgres_type: "{{ pg_config['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}"
+
+- name: Default label selector to custom resource generated postgres
+  set_fact:
+    postgres_label_selector: "app.kubernetes.io/name={{ deployment_name }}-postgres"
+  when: postgres_label_selector is not defined
+
+- name: Get the postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: '{{ meta.namespace }}'
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+  register: postgres_pod
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  delay: 5
+  retries: 60
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+
+- name: Determine the timestamp for the backup once for all nodes
+  set_fact:
+    now: '{{ lookup("pipe", "date +%F-%T") }}'
+
+- name: Set backup directory name
+  set_fact:
+    backup_dir: "/backups/tower-openshift-backup-{{ now }}"
+
+- name: Create directory for backup
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      mkdir -p {{ backup_dir }}
+
+- name: Precreate file for database dump
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      touch {{ backup_dir }}/tower.db
+
+- name: Set permissions on file for database dump
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db"
+
+- name: Set full resolvable host name for postgres pod
+  set_fact:
+    resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local"
+  when: awx_postgres_type == 'managed'
+
+- name: Set pg_dump command
+  set_fact:
+    pgdump: >-
+      pg_dump --clean --create
+      -h {{ resolvable_db_host }}
+      -U {{ awx_postgres_user }}
+      -d {{ awx_postgres_database }}
+      -p {{ awx_postgres_port }}
+
+- name: Write pg_dump to backup on PVC
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ backup_dir }}/tower.db"
+  register: data_migration

roles/backup/tasks/secrets.yml

@@ -0,0 +1,61 @@
+---
+
+- name: Get secret_key
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: "{{ this_awx['resources'][0]['status']['towerSecretKeySecret'] }}"
+  register: _secret_key
+
+- name: Set secret key
+  set_fact:
+    secret_key: "{{ _secret_key['resources'][0]['data']['secret_key'] | b64decode }}"
+
+- name: Get admin_password
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: "{{ this_awx['resources'][0]['status']['towerAdminPasswordSecret'] }}"
+  register: _admin_password
+
+- name: Set admin_password
+  set_fact:
+    admin_password: "{{ _admin_password['resources'][0]['data']['password'] | b64decode }}"
+
+- name: Get broadcast_websocket
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: "{{ this_awx['resources'][0]['status']['towerBroadcastWebsocketSecret'] }}"
+  register: _broadcast_websocket
+
+- name: Set broadcast_websocket key
+  set_fact:
+    broadcast_websocket: "{{ _broadcast_websocket['resources'][0]['data']['secret'] | b64decode }}"
+
+- name: Get postgres configuration
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: "{{ this_awx['resources'][0]['status']['towerPostgresConfigurationSecret'] }}"
+  register: _postgres_configuration
+
+- name: Set postgres configuration
+  set_fact:
+    database_password: "{{ _postgres_configuration['resources'][0]['data']['password'] | b64decode }}"
+    database_username: "{{ _postgres_configuration['resources'][0]['data']['username'] | b64decode }}"
+    database_name: "{{ _postgres_configuration['resources'][0]['data']['database'] | b64decode }}"
+    database_port: "{{ _postgres_configuration['resources'][0]['data']['port'] | b64decode }}"
+    database_host: "{{ _postgres_configuration['resources'][0]['data']['host'] | b64decode }}"
+    database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}"
+
+- name: Template secrets into yaml
+  set_fact:
+    secrets_file: "{{ lookup('template', 'secrets.yml.j2') }}"
+
+- name: Write postgres configuration to pvc
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "echo '{{ secrets_file }}' > {{ backup_dir }}/secrets.yml"

roles/backup/tasks/update_status.yml

@@ -0,0 +1,13 @@
+---
+
+# The backup directory in this status can be referenced when restoring
+- name: Update Tower Backup status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      backupDirectory: "{{ backup_dir }}"
+      backupClaim: "{{ backup_claim }}"
+  when: backup_complete

roles/backup/templates/backup_pvc.yml.j2

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ deployment_name }}-backup-claim
+  namespace: {{ backup_pvc_namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+{% if backup_storage_class != '' %}
+  storageClassName: {{ backup_storage_class }}
+{% endif %}
+  resources:
+    requests:
+      storage: {{ backup_storage_requirements | default('5Gi', true) }}

roles/backup/templates/event.yml.j2

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Event
+metadata:
+  name: backup-error.{{ now }}
+  namespace: {{ meta.namespace }}
+involvedObject:
+  apiVersion: awx.ansible.com/v1beta1
+  kind: {{ kind }}
+  name: {{ meta.name }}
+  namespace: {{ meta.namespace }}
+message: {{ error_msg }}
+reason: BackupFailed
+type: Warning
+firstTimestamp: {{ now }}
+lastTimestamp: {{ now }}
+count: 1

roles/backup/templates/management-pod.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ meta.name }}-db-management
+  namespace: {{ backup_pvc_namespace }}
+spec:
+  containers:
+  - name: {{ meta.name }}-db-management
+    image: "{{ postgres_image }}"
+    imagePullPolicy: Always
+    command: ["sleep", "infinity"]
+    volumeMounts:
+    - name: {{ meta.name }}-backup
+      mountPath: /backups
+      readOnly: false
+  volumes:
+    - name: {{ meta.name }}-backup
+      persistentVolumeClaim:
+        claimName: {{ backup_claim }}
+        readOnly: false
+  restartPolicy: Never

roles/backup/templates/secrets.yml.j2

@@ -0,0 +1,10 @@
+---
+secret_key: {{ secret_key }}
+admin_password: {{ admin_password }}
+broadcast_websocket: {{ broadcast_websocket }}
+database_password: {{ database_password }}
+database_username: {{ database_username }}
+database_name: {{ database_name }}
+database_port: {{ database_port }}
+database_host: {{ database_host }}
+database_type: {{ database_type }}

roles/backup/vars/main.yml

@@ -0,0 +1,4 @@
+---
+deployment_type: "awx"
+postgres_image: postgres:12
+backup_complete: false

roles/installer/tasks/database_configuration.yml

@@ -64,6 +64,10 @@
   set_fact:
     pg_config: '{{ _generated_pg_config_resources["resources"] | default([]) | length | ternary(_generated_pg_config_resources, _pg_config) }}'
 
+- name: Set actual postgres configuration secret used
+  set_fact:
+    postgres_configuration_secret: "{{ pg_config['resources'][0]['metadata']['name'] }}"
+
 - block:
     - name: Create Database if no database is specified
       k8s:
@@ -100,7 +104,6 @@
         definition: "{{ lookup('template', 'tower_postgres.yaml.j2') }}"
   when: pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed'
 
-
 - name: Store Database Configuration
   set_fact:
     awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
@@ -112,8 +115,8 @@
 
 - name: Look up details for this deployment
   k8s_info:
-    api_version: 'v1beta1'  # TODO: How to parameterize this?
-    kind: "AWX"  #  TODO: How to parameterize this?
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
     name: "{{ meta.name }}"
     namespace: "{{ meta.namespace }}"
   register: this_awx

roles/installer/tasks/initialize_django.yml

@@ -1,20 +1,31 @@
 ---
 - name: Check if there are any super users defined.
-  community.kubernetes.k8s_exec:
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"
     command: >-
       bash -c "echo 'from django.contrib.auth.models import User;
-      nsu = User.objects.filter(is_superuser=True).count();
+      nsu = User.objects.filter(is_superuser=True, username='{{ tower_admin_user }}').count();
       exit(0 if nsu > 0 else 1)'
       | awx-manage shell"
   ignore_errors: true
   register: users_result
   changed_when: users_result.return_code > 0
 
+- name: Update super user password via Django if it does exist (same password is a noop)
+  k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ tower_pod_name }}"
+    container: "{{ meta.name }}-task"
+    command: >-
+      bash -c "awx-manage update_password --username '{{ tower_admin_user }}' --password '{{ tower_admin_password }}'"
+  register: update_pw_result
+  changed_when: users_result.stdout == 'Password not updated'
+  when: users_result.return_code == 0
+
 - name: Create super user via Django if it doesn't exist.
-  community.kubernetes.k8s_exec:
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"
@@ -25,7 +36,7 @@
   when: users_result.return_code > 0
 
 - name: Create preload data if necessary.  # noqa 305
-  community.kubernetes.k8s_exec:
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"

roles/installer/tasks/load_ldap_cacert_secret.yml

@@ -1,6 +1,6 @@
 ---
 - name: Retrieve LDAP CA Certificate Secret
-  community.kubernetes.k8s_info:
+  k8s_info:
     kind: Secret
     namespace: '{{ meta.namespace }}'
     name: '{{ ldap_cacert_secret }}'

roles/installer/tasks/load_route_tls_secret.yml

@@ -1,6 +1,6 @@
 ---
 - name: Retrieve Route TLS Secret
-  community.kubernetes.k8s_info:
+  k8s_info:
     kind: Secret
     namespace: '{{ meta.namespace }}'
     name: '{{ tower_route_tls_secret }}'

roles/installer/tasks/migrate_data.yml

@@ -1,12 +1,21 @@
 ---
 
+- name: Set actual old postgres configuration secret name
+  set_fact:
+    old_postgres_configuration_name: "{{ old_pg_config['resources'][0]['metadata']['name'] }}"
+
 - name: Store Database Configuration
   set_fact:
-    tower_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}"
-    tower_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}"
-    tower_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}"
-    tower_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}"
-    tower_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}"
+    awx_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}"
+    awx_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}"
+    awx_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}"
+    awx_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}"
+    awx_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}"
+
+- name: Default label selector to custom resource generated postgres
+  set_fact:
+    postgres_label_selector: "app.kubernetes.io/name={{ meta.name }}-postgres"
+  when: postgres_label_selector is not defined
 
 - name: Get the postgres pod information
   k8s_info:
@@ -16,7 +25,9 @@
     field_selectors:
       - status.phase=Running
   register: postgres_pod
-  until: postgres_pod['resources'] | length
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
   delay: 5
   retries: 60
 
@@ -31,10 +42,10 @@
   set_fact:
     pgdump: >-
       pg_dump --clean --create
-      -h {{ tower_old_postgres_host }}
-      -U {{ tower_old_postgres_user }}
-      -d {{ tower_old_postgres_database }}
-      -p {{ tower_old_postgres_port }}
+      -h {{ awx_old_postgres_host }}
+      -U {{ awx_old_postgres_user }}
+      -d {{ awx_old_postgres_database }}
+      -p {{ awx_old_postgres_port }}
 
 - name: Set pg_restore command
   set_fact:
@@ -44,13 +55,13 @@
       -p {{ awx_postgres_port }}
 
 - name: Stream backup from pg_dump to the new postgresql container
-  community.kubernetes.k8s_exec:
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ postgres_pod_name }}"
     command: |
       bash -c """
       set -e -o pipefail
-      PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }}
+      PGPASSWORD={{ awx_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }}
       echo 'Successful'
       """
   register: data_migration
@@ -58,4 +69,4 @@
 
 - name: Set flag signifying that this instance has been migrated
   set_fact:
-    tower_migrated_from_secret: "{{ tower_old_postgres_configuration_secret }}"
+    tower_migrated_from_secret: "{{ old_postgres_configuration_name }}"

roles/installer/tasks/update_status.yml

@@ -17,8 +17,35 @@
     status:
       towerAdminUser: "{{ tower_admin_user }}"
 
+- name: Update postgres configuration status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      towerPostgresConfigurationSecret: "{{ pg_config['resources'][0]['metadata']['name'] }}"
+
+- name: Update broadcast websocket status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      towerBroadcastWebsocketSecret: "{{ broadcast_websocket_secret['resources'][0]['metadata']['name'] }}"
+
+- name: Update secret key status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      towerSecretKeySecret: "{{ secret_key_secret_name }}"
+
 - name: Retrieve instance version
-  community.kubernetes.k8s_exec:
+  k8s_exec:
     namespace: "{{ meta.namespace }}"
     pod: "{{ tower_pod_name }}"
     container: "{{ meta.name }}-task"
@@ -47,7 +74,7 @@
 
 - block:
     - name: Retrieve route URL
-      community.kubernetes.k8s_info:
+      k8s_info:
         kind: Route
         namespace: '{{ meta.namespace }}'
         name: '{{ meta.name }}'

roles/installer/templates/tower_postgres.yaml.j2

@@ -53,17 +53,17 @@ spec:
             - name: POSTGRES_DB
               valueFrom:
                 secretKeyRef:
-                  name: '{{ meta.name }}-postgres-configuration'
+                  name: '{{ postgres_configuration_secret }}'
                   key: database
             - name: POSTGRES_USER
               valueFrom:
                 secretKeyRef:
-                  name: '{{ meta.name }}-postgres-configuration'
+                  name: '{{ postgres_configuration_secret }}'
                   key: username
             - name: POSTGRES_PASSWORD
               valueFrom:
                 secretKeyRef:
-                  name: '{{ meta.name }}-postgres-configuration'
+                  name: '{{ postgres_configuration_secret }}'
                   key: password
             - name: PGDATA
               value: '{{ tower_postgres_data_path }}'
@@ -72,7 +72,7 @@ spec:
             - name: POSTGRES_HOST_AUTH_METHOD
               value: '{{ postgres_host_auth_method }}'
           ports:
-            - containerPort: 5432
+            - containerPort: {{ awx_postgres_port | default('5432')}}
               name: postgres
           volumeMounts:
             - name: postgres

roles/restore/README.md

@@ -0,0 +1,121 @@
+Restore Role
+=========
+
+The purpose of this role is to restore your AWX deployment from an existing PVC backup. The backup includes:
+  - custom deployment specific values in the spec section of the AWX custom resource object
+  - backup of the postgresql database
+  - secret_key, admin_password, and broadcast_websocket secrets
+  - database configuration
+
+
+
+Requirements
+------------
+
+This role assumes you are authenticated with an Openshift or Kubernetes cluster:
+  - The awx-operator has been deployed to the cluster
+  - AWX is deployed to via the operator
+  - An AWX backup is available on a PVC in your cluster (see the backup [README.md](../backup/README.md))
+
+
+Usage
+----------------
+
+Then create a file named `restore-awx.yml` with the following contents:
+
+```yaml
+---
+apiVersion: awx.ansible.com/v1beta1
+kind: AWXRestore
+metadata:
+  name: restore1
+  namespace: my-namespace
+spec:
+  deployment_name: mytower
+  backup: awxbackup-2021-04-22
+  backup_pvc_namespace: 'old-awx-namespace'
+```
+
+Note that the `deployment_name` above is the name of the AWX deployment you intend to create and restore to.  
+
+The namespace specified is the namespace the resulting AWX deployment will be in.  The namespace you specified must be pre-created.  
+
+```
+kubectl create ns my-namespace
+```
+
+Finally, use `kubectl` to create the restore object in your cluster:
+
+```bash
+$ kubectl apply -f restore-awx.yml
+```
+
+This will create a new deployment and restore your backup to it.
+
+> :warning: tower_admin_password_secret value will replace the password for the `tower_admin_user` user (by default, this is the `admin` user).
+
+
+Role Variables
+--------------
+
+The name of the backup directory can be found as a status on your AWXBackup object.  This can be found in your cluster's console, or with the client as shown below.  
+
+```bash
+$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.backupDirectory}"
+/backups/tower-openshift-backup-2021-04-02-03:25:08
+```
+
+```
+backup_dir: '/backups/tower-openshift-backup-2021-04-02-03:25:08'
+```
+
+
+The name of the PVC can also be found by looking at the backup object.  
+
+```bash
+$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.backupClaim}"
+awx-backup-volume-claim
+```
+
+```
+backup_pvc: 'awx-backup-volume-claim'
+```
+
+By default, the backup pvc will be created in the same namespace the awxbackup object is created in. This namespace must be specified using the `backup_pvc_namespace` variable.
+
+```
+backup_pvc_namespace: 'custom-namespace'
+```
+
+If a custom postgres configuration secret was used when deploying AWX, it must be set:
+
+```
+tower_postgres_configuration_secret: 'awx-postgres-configuration'
+```
+
+If the awxbackup object no longer exists, it is still possible to restore from the backup it created by specifying the pvc name and the back directory.
+
+```
+backup_pvc: myoldtower-backup-claim
+backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08
+```
+
+
+Testing
+----------------
+
+You can test this role directly by creating and running the following playbook with the appropriate variables:
+
+```
+---
+- name: Restore AWX
+  hosts: localhost
+  gather_facts: false
+  roles:
+    - restore
+```
+
+License
+-------
+
+MIT

roles/restore/defaults/main.yml

@@ -0,0 +1,14 @@
+---
+# Required: specify name of tower deployment to restore to
+deployment_name: ''
+kind: 'AWXRestore'
+api_version: '{{ deployment_type }}.ansible.com/v1beta1'
+
+# Required: specify a pre-created PVC (name) to restore from
+backup_pvc: ''
+backup_pvc_namespace: ''
+
+# Required: backup name, found on the awxbackup object
+backup_dir: ''
+
+tower_postgres_configuration_secret: "{{ deployment_name }}-postgres-configuration"

roles/restore/meta/main.yml

@@ -0,0 +1,31 @@
+---
+galaxy_info:
+  author: Ansible
+  description: AWX role for AWX Operator for Kubernetes.
+  company: Red Hat, Inc.
+
+  license: MIT
+
+  min_ansible_version: 2.8
+
+  platforms:
+    - name: EL
+      versions:
+        - all
+    - name: Debian
+      versions:
+        - all
+
+  galaxy_tags:
+    - tower
+    - controller
+    - awx
+    - ansible
+    - restore
+    - automation
+
+dependencies: []
+
+collections:
+  - community.kubernetes
+  - operator_sdk.util

roles/restore/tasks/cleanup.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Delete any existing management pod
+  k8s:
+    name: "{{ meta.name }}-db-management"
+    kind: Pod
+    namespace: "{{ backup_pvc_namespace }}"
+    state: absent
+    force: true
+
+- name: Remove ownerReferences from secrets to avoid garbage collection
+  k8s:
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: '{{ item }}'
+        namespace: '{{ meta.namespace }}'
+        ownerReferences: null
+  loop:
+    - '{{ deployment_name }}-admin-password'
+    - '{{ deployment_name }}-secret-key'
+    - '{{ deployment_name }}-postgres-configuration'
+    - '{{ deployment_name }}-broadcast-websocket'

roles/restore/tasks/deploy_awx.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Get AWX object definition from pvc
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "cat '{{ backup_dir }}/awx_object'"
+  register: awx_object
+
+- name: Set AWX spec variable from backup
+  set_fact:
+    awx_spec: "{{ awx_object.stdout }}"
+
+- name: Deploy AWX
+  k8s:
+    state: "{{ state | default('present') }}"
+    namespace: "{{ meta.namespace }}"
+    apply: yes
+    template: awx_object.yml.j2
+    wait: true
+    wait_condition:
+      type: "Running"
+      status: "True"
+
+# TODO: Add logic to allow users to provide override values here,
+#       or to specify spec values that were not in the backed up AWX object.
+#       This may involve changing how we back up the spec section of the AWX object
+
+- name: Remove ownerReferences to prevent garbage collection of new AWX CRO
+  k8s:
+    definition:
+      apiVersion: '{{ api_version }}'
+      kind: AWX
+      metadata:
+        name: '{{ deployment_name }}'
+        namespace: '{{ meta.namespace }}'
+        ownerReferences: null

roles/restore/tasks/error_handling.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Determine the timestamp
+  set_fact:
+    now: '{{ lookup("pipe", "date +%FT%TZ") }}'
+
+- name: Emit ocp event with error
+  k8s:
+    kind: Event
+    namespace: "{{ meta.namespace }}"
+    template: "event.yml.j2"

roles/restore/tasks/init.yml

@@ -0,0 +1,88 @@
+---
+
+- name: Set variables from awxbackup object statuses if provided
+  block:
+  - name: Look up details for the backup object
+    k8s_info:
+      api_version: "{{ api_version }}"
+      kind: "AWXBackup"
+      name: "{{ backup }}"
+      namespace: "{{ backup_pvc_namespace }}"
+    register: this_backup
+
+  - name: Set backup pvc name from status
+    set_fact:
+      backup_pvc: "{{ this_backup['resources'][0]['status']['backupClaim'] }}"
+
+  - name: Set tmp backup directory from status
+    set_fact:
+      backup_dir: "{{ this_backup['resources'][0]['status']['backupDirectory'] }}"
+  when:
+    - backup != '' or backup is defined
+
+# Check to make sure provided pvc exists, error loudly if not.  Otherwise, the management pod will just stay in pending state forever.
+- name: Check provided PVC exists
+  k8s_info:
+    name: "{{ backup_pvc }}"
+    kind: PersistentVolumeClaim
+    namespace: "{{ backup_pvc_namespace }}"
+  register: provided_pvc
+  when:
+    - backup_pvc != ''
+
+- name: Surface error to user
+  block:
+    - name: Set error message
+      set_fact:
+        error_msg: "{{ backup_pvc }} does not exist, please create this pvc first."
+
+    - name: Handle error
+      import_tasks: error_handling.yml
+
+    - name: Fail early if pvc is defined but does not exist
+      fail:
+        msg: "{{ error_msg }}"
+  when:
+    - backup_pvc != ''
+    - provided_pvc.resources | length == 0
+
+- name: Delete any existing management pod
+  k8s:
+    name: "{{ meta.name }}-db-management"
+    kind: Pod
+    namespace: "{{ backup_pvc_namespace }}"
+    state: absent
+    force: true
+    wait: true
+
+- name: Create management pod from templated deployment config
+  k8s:
+    name: "{{ meta.name }}-db-management"
+    kind: Deployment
+    state: present
+    template: "management-pod.yml.j2"
+    wait: true
+
+- name: Check to make sure backup directory exists on PVC
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "stat {{ backup_dir }}"
+  register: stat_backup_dir
+
+- name: Error if backup dir is missing
+  block:
+    - name: Set error message
+      set_fact:
+        error_msg: "{{ backup_dir }} does not exist, see the backupDirectory status on your AWXBackup for the correct backup_dir."
+
+    - name: Handle error
+      import_tasks: error_handling.yml
+
+    - name: Fail early if backup dir provided does not exist
+      fail:
+        msg: "{{ error_msg }}"
+  when:
+    - backup_dir != ''
+    - stat_backup_dir.return_code != 0

roles/restore/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+
+- name: Look up details for this restore object
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+  register: this_restore
+
+- block:
+    - include_tasks: init.yml
+
+    - include_tasks: secrets.yml
+
+    - include_tasks: deploy_awx.yml
+
+    - include_tasks: postgres.yml
+
+    - name: Set flag signifying this restore was successful
+      set_fact:
+        tower_restore_complete: True
+
+    - include_tasks: cleanup.yml
+
+  when:
+    - this_restore['resources'][0]['status']['towerRestoreComplete'] is not defined
+
+- name: Update status variables
+  include_tasks: update_status.yml
+
+# TODO: backup tower settings or make sure that users only specify settings/config changes via AWX object.  See ticket

roles/restore/tasks/postgres.yml

@@ -0,0 +1,95 @@
+---
+
+- name: Check for specified PostgreSQL configuration
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: '{{ tower_postgres_configuration_secret }}'
+  register: _custom_pg_config_resources
+  when: tower_postgres_configuration_secret | length
+
+- name: Check for default PostgreSQL configuration
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: '{{ deployment_name }}-postgres-configuration'
+  register: _default_pg_config_resources
+
+- name: Set PostgreSQL configuration
+  set_fact:
+    pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}'
+
+- name: Store Database Configuration
+  set_fact:
+    awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
+    awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}"
+    awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
+    awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
+    awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
+    awx_postgres_type: "{{ pg_config['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}"
+
+- name: Default label selector to custom resource generated postgres
+  set_fact:
+    postgres_label_selector: "app.kubernetes.io/name={{ deployment_name }}-postgres"
+  when: postgres_label_selector is not defined
+
+- name: Get the postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: '{{ meta.namespace }}'
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+  register: postgres_pod
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  delay: 5
+  retries: 60
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+
+- name: Check for presence of AWX Deployment
+  k8s_info:
+    api_version: v1
+    kind: Deployment
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+  register: this_deployment
+
+- name: Scale down Deployment for migration
+  k8s_scale:
+    api_version: v1
+    kind: Deployment
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    replicas: 0
+  when: this_deployment['resources'] | length
+
+- name: Set full resolvable host name for postgres pod
+  set_fact:
+    resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local"
+  when: awx_postgres_type == 'managed'
+
+- name: Set pg_restore command
+  set_fact:
+    psql_restore: >-
+      psql -U {{ awx_postgres_user }}
+      -h {{ resolvable_db_host }}
+      -U {{ awx_postgres_user }}
+      -d {{ awx_postgres_database }}
+      -p {{ awx_postgres_port }}
+
+- name: Restore database dump to the new postgresql container
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: |
+      bash -c """
+      set -e -o pipefail
+      cat {{ backup_dir }}/tower.db | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }}
+      echo 'Successful'
+      """
+  register: data_migration
+  failed_when: "'Successful' not in data_migration.stdout"

roles/restore/tasks/secrets.yml

@@ -0,0 +1,37 @@
+---
+
+- name: Get secret definition from pvc
+  k8s_exec:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "cat '{{ backup_dir }}/secrets.yml'"
+  register: secrets
+
+- name: Create temp vars file
+  tempfile:
+    prefix: secret_vars-
+  register: secret_vars
+
+- name: Write vars to file locally
+  copy:
+    dest: "{{ secret_vars.path }}"
+    content: "{{ secrets.stdout }}"
+    mode: 0640
+
+- name: Include secret vars from backup
+  include_vars: "{{ secret_vars.path }}"
+
+- name: Set new database host based on supplied deployment_name
+  set_fact:
+    database_host: "{{ deployment_name }}-postgres"
+  when:
+    - database_type == 'managed'
+
+- name: Apply secret
+  k8s:
+    state: present
+    namespace: "{{ meta.namespace }}"
+    apply: yes
+    wait: yes
+    template: "secrets.yml.j2"

roles/restore/tasks/update_status.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Update Tower Restore status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      towerRestoreComplete: true
+  when: tower_restore_complete is defined

roles/restore/templates/awx_object.yml.j2

@@ -0,0 +1,7 @@
+---
+apiVersion: '{{ api_version }}'
+kind: AWX
+metadata:
+  name: '{{ deployment_name }}'
+  namespace: '{{ meta.namespace }}'
+spec: {{ awx_spec }}

roles/restore/templates/event.yml.j2

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Event
+metadata:
+  name: restore-error.{{ now }}
+  namespace: {{ meta.namespace }}
+involvedObject:
+  apiVersion: awx.ansible.com/v1beta1
+  kind: {{ kind }}
+  name: {{ meta.name }}
+  namespace: {{ meta.namespace }}
+message: {{ error_msg }}
+reason: RestoreFailed
+type: Warning
+firstTimestamp: {{ now }}
+lastTimestamp: {{ now }}
+count: 1

roles/restore/templates/management-pod.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ meta.name }}-db-management
+  namespace: {{ backup_pvc_namespace }}
+spec:
+  containers:
+  - name: {{ meta.name }}-db-management
+    image: "{{ postgres_image }}"
+    imagePullPolicy: Always
+    command: ["sleep", "infinity"]
+    volumeMounts:
+    - name: {{ meta.name }}-backup
+      mountPath: /backups
+      readOnly: false
+  volumes:
+    - name: {{ meta.name }}-backup
+      persistentVolumeClaim:
+        claimName: {{ backup_pvc }}
+        readOnly: false
+  restartPolicy: Never

roles/restore/templates/secrets.yml.j2

@@ -0,0 +1,44 @@
+# Postgres Secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ deployment_name }}-postgres-configuration'
+  namespace: '{{ meta.namespace }}'
+stringData:
+  password: '{{ database_password }}'
+  username: '{{ database_username }}'
+  database: '{{ database_name }}'
+  port: '{{ database_port }}'
+  host: '{{ database_host }}'
+  type: '{{ database_type }}'
+
+# Secret Key Secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ deployment_name }}-secret-key'
+  namespace: '{{ meta.namespace }}'
+stringData:
+  secret_key: '{{ secret_key }}'
+
+# Admin Password Secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ deployment_name }}-admin-password'
+  namespace: '{{ meta.namespace }}'
+stringData:
+  password: '{{ admin_password }}'
+
+# Broadcast Websocket Secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ deployment_name }}-broadcast-websocket'
+  namespace: '{{ meta.namespace }}'
+stringData:
+  secret: '{{ broadcast_websocket }}'

roles/restore/vars/main.yml

@@ -0,0 +1,4 @@
+---
+
+deployment_type: "awx"
+postgres_image: postgres:12

watches.yaml

@@ -6,3 +6,13 @@
   finalizer:
     name: finalizer.awx.ansible.com
     role: finalizer
+
+- version: v1beta1
+  group: awx.ansible.com
+  kind: AWXBackup
+  role: backup
+
+- version: v1beta1
+  group: awx.ansible.com
+  kind: AWXRestore
+  role: restore