Fix backup reconciliation loop, add error status

deploy/crds/awx.ansible.com_awxbackups_crd.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
@@ -11,12 +12,21 @@ spec:
     singular: awxbackup
   scope: Namespaced
   versions:
-  - name: v1beta1
-    schema:
-      openAPIV3Schema:
-        type: object
-        x-kubernetes-preserve-unknown-fields: true
-    served: true
-    storage: true
-    subresources:
-      status: {}
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the AWXBackup CRD
+          # TODO: Figure out how to require the tower_name field
+          properties:
+            spec:
+              type: object
+              properties:
+                tower_name:
+                  description: Name of the deployment to be backed up
+                  type: string

deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml

@@ -1,9 +1,11 @@
+---
 apiVersion: awx.ansible.com/v1beta1
 kind: AWXBackup
 metadata:
   name: example-awxbackup
   namespace: example-awx
 spec:
+  tower_name: ''
   tower_backup_pvc: ''
   tower_backup_size: ''
   tower_backup_storage_class: ''

roles/backup/README.md

@@ -28,9 +28,12 @@ kind: AWXBackup
 metadata:
   name: awxbackup
   namespace: my-namespace
+spec:
+  tower_name: mytower
 ```
 
-> The metadata.name you provide, is the name of the AWX deployment you intend to backup from (in case you have multiple in the same namespace).
+Note that the `tower_name` above is the name of the AWX deployment you intend to backup from.  
+
 
 Finally, use `kubectl` to create the backup object in your cluster:
 

roles/backup/defaults/main.yml

@@ -1,4 +1,6 @@
 ---
+# Required: specify name of tower deployment to backup from
+tower_name: ''
 
 # Specify a pre-created PVC (name) to backup to
 tower_backup_pvc: ''
@@ -10,7 +12,7 @@ tower_backup_size: ''
 tower_backup_storage_class: ''
 
 # Secret Names
-tower_secret_key_secret: "{{ meta.name }}-secret-key"
-tower_admin_password_secret: "{{ meta.name }}-admin-password"
-tower_broadcast_websocket_secret: "{{ meta.name }}-broadcast-websocket"
-tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration"
+tower_secret_key_secret: "{{ tower_name }}-secret-key"
+tower_admin_password_secret: "{{ tower_name }}-admin-password"
+tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket"
+tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration"

roles/backup/handlers/main.yml

@@ -0,0 +1,17 @@
+---
+
+- name: Update awxbackup status
+  block:
+  - name: Set apiVersion and kind variables
+    set_fact:
+      api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
+      kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
+
+  - name: Update error status
+    operator_sdk.util.k8s_status:
+      api_version: '{{ api_version }}'
+      kind: "{{ kind }}"
+      name: "{{ meta.name }}"
+      namespace: "{{ meta.namespace }}"
+      status:
+        error: "{{ error_msg }}"

roles/backup/store_secrets.yml

@@ -1,7 +0,0 @@
----
-
-- name: Test Getting and Storing Secrets (OCP)
-  hosts: localhost
-
-  tasks:
-  - import_tasks: tasks/secrets.yml

roles/backup/tasks/awx-cro.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Get AWX custom resource object
+  k8s_info:
+    version: v1beta1
+    kind: AWX
+    namespace: '{{ meta.namespace }}'
+    name: '{{ tower_name }}'
+  register: _awx_cro
+
+- name: Set AWX object
+  set_fact:
+    _awx: "{{ _awx_cro['resources'][0] }}"
+
+- name: Set apiVersion
+  set_fact:
+    awx_api_version: "{{ _awx['apiVersion'] }}"
+
+- name: Set user specified spec
+  set_fact:
+    awx_spec: "{{ _awx['spec'] }}"
+
+- name: Template AWX object definition
+  template:
+    src: awx_object.yml.j2
+    dest: "_secrets/awx_object.yml"
+    mode: '0600'
+
+- name: Set AWX object template file as var
+  set_fact:
+    awx_object_template: "{{ lookup('file', '_secrets/awx_object.yml') }}"
+
+- name: Write awx object to pvc
+  community.kubernetes.k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "echo '{{ awx_object_template }}' > {{ _backup_dir }}/awx_object.yml"

roles/backup/tasks/init.yml

@@ -19,6 +19,11 @@
   when:
     - tower_backup_pvc != '' or tower_backup_pvc is defined
 
+- name: Update status
+  set_fact:
+    error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first."
+  notify: "Update awxbackup status"
+
 - name: Fail early if pvc is defined but does not exist
   fail:
     msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first."
@@ -29,12 +34,10 @@
   set_fact:
     _default_backup_pvc: "{{ meta.name }}-backup-claim"
 
+# by default, it will re-use the old pvc if already created (unless pvc is provided)
 - name: Set PVC to use for backup
   set_fact:
-    backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true)}}"
-
-# TODO: re-use the old pvc if already created (unless pvc is provided)
-# TODO: allow users to configure their own storage class for dynamically creating a pvc?
+    backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}"
 
 - name: Create PVC for backup
   community.kubernetes.k8s:

roles/backup/tasks/main.yml

@@ -1,26 +1,37 @@
 ---
 
+- name: Set apiVersion and kind variables
+  set_fact:
+    api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
+    kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
+
 - name: Look up details for this deployment
   k8s_info:
-    api_version: 'v1beta1'  # TODO: How to parameterize this?
-    kind: "AWX"  #  TODO: How to parameterize this?
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
     name: "{{ meta.name }}"
     namespace: "{{ meta.namespace }}"
-  register: this_awx
+  register: this_backup
 
 - block:
-  - include_tasks: init.yml
+    - include_tasks: init.yml
 
-  - include_tasks: postgres.yml
+    - include_tasks: postgres.yml
 
-  - include_tasks: secrets.yml
+    - include_tasks: secrets.yml
 
-  # TODO: Add task to change the status on the backup CR when this runs successfully
-  - name: Set flag signifying this backup was successful
-    set_fact:
-      tower_backup_complete: "{{ _backup_dir }}"
+    - include_tasks: awx-cro.yml
 
-  - include_tasks: cleanup.yml
+    - name: Set flag signifying this backup was successful
+      set_fact:
+        tower_backup_complete: "{{ _backup_dir }}"
+
+    - include_tasks: cleanup.yml
 
   when:
-    - this_awx['resources'][0]['status']['towerMigratedFromSecret'] is not defined
+    - this_backup['resources'][0]['status']['towerBackupComplete'] is not defined
+
+- name: Update status variables
+  include_tasks: update_status.yml
+
+# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object.  See ticket

roles/backup/tasks/postgres.yml

@@ -1,91 +1,89 @@
 ---
-  - name: Check for specified PostgreSQL configuration
-    k8s_info:
-      kind: Secret
-      namespace: '{{ meta.namespace }}'
-      name: '{{ tower_postgres_configuration_secret }}'
-    register: _custom_pg_config_resources
-    when: tower_postgres_configuration_secret | length
 
-  - name: Check for default PostgreSQL configuration
-    k8s_info:
-      kind: Secret
-      namespace: '{{ meta.namespace }}'
-      name: '{{ meta.name }}-postgres-configuration'
-    register: _default_pg_config_resources
+- name: Check for specified PostgreSQL configuration
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: '{{ tower_postgres_configuration_secret }}'
+  register: _custom_pg_config_resources
+  when: tower_postgres_configuration_secret | length
 
-  - name: Set PostgreSQL configuration
-    set_fact:
-      pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}'
+- name: Check for default PostgreSQL configuration
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: '{{ tower_name }}-postgres-configuration'
+  register: _default_pg_config_resources
 
-  - name: Store Database Configuration
-    set_fact:
-      awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
-      awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}"
-      awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
-      awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
-      awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
+- name: Set PostgreSQL configuration
+  set_fact:
+    pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}'
 
-  - name: Get the postgres pod information
-    k8s_info:
-      kind: Pod
-      namespace: '{{ meta.namespace }}'
-      label_selectors:
-        - "app={{ meta.name }}-{{ deployment_type }}-postgres"
-    register: postgres_pod
-    until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
-    delay: 5
-    retries: 60
+- name: Store Database Configuration
+  set_fact:
+    awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
+    awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}"
+    awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
+    awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
+    awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
 
-  - name: Set the resource pod name as a variable.
-    set_fact:
-      postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+- name: Get the postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: '{{ meta.namespace }}'
+    label_selectors:
+      - "app={{ tower_name }}-{{ deployment_type }}-postgres"
+  register: postgres_pod
+  until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  delay: 5
+  retries: 60
 
-  - name: Determine the timestamp for the backup once for all nodes
-    set_fact:
-      now: '{{ lookup("pipe", "date +%F-%T") }}'
+- name: Set the resource pod name as a variable.
+  set_fact:
+    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
 
-  - name: Set backup directory name
-    set_fact:
-      _backup_dir: "/backups/tower-openshift-backup-{{ now }}"
+- name: Determine the timestamp for the backup once for all nodes
+  set_fact:
+    now: '{{ lookup("pipe", "date +%F-%T") }}'
 
-  - name: Create directory for backup
-    community.kubernetes.k8s_exec:
-      namespace: "{{ meta.namespace }}"
-      pod: "{{ meta.name }}-db-management"
-      command: >-
-        mkdir -p {{ _backup_dir }}
+- name: Set backup directory name
+  set_fact:
+    _backup_dir: "/backups/tower-openshift-backup-{{ now }}"
 
-  - name: Precreate file for database dump
-    community.kubernetes.k8s_exec:
-      namespace: "{{ meta.namespace }}"
-      pod: "{{ meta.name }}-db-management"
-      command: >-
-        touch {{ _backup_dir }}/tower.db
+- name: Create directory for backup
+  community.kubernetes.k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      mkdir -p {{ _backup_dir }}
 
-  - name: Set permissions on file for database dump
-    community.kubernetes.k8s_exec:
-      namespace: "{{ meta.namespace }}"
-      pod: "{{ meta.name }}-db-management"
-      command: >-
-        chmod 0600 {{ _backup_dir }}/tower.db
+- name: Precreate file for database dump
+  community.kubernetes.k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      touch {{ _backup_dir }}/tower.db
 
-  - name: Set pg_dump command
-    set_fact:
-      pgdump: >-
-        pg_dump --clean --create
-        -h {{ awx_postgres_host }}
-        -U {{ awx_postgres_user }}
-        -d {{ awx_postgres_database }}
-        -p {{ awx_postgres_port }}
+- name: Set permissions on file for database dump
+  community.kubernetes.k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      chmod 0600 {{ _backup_dir }}/tower.db
 
-  - name: Write pg_dump to backup on PVC
-    community.kubernetes.k8s_exec:
-      namespace: "{{ meta.namespace }}"
-      pod: "{{ meta.name }}-db-management"
-      command: >-
-        bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db"
-    register: data_migration
+- name: Set pg_dump command
+  set_fact:
+    pgdump: >-
+      pg_dump --clean --create
+      -h {{ awx_postgres_host }}
+      -U {{ awx_postgres_user }}
+      -d {{ awx_postgres_database }}
+      -p {{ awx_postgres_port }}
 
-  # TODO: Backup secret key and other secrets - look at trad tower backup pattern
-  # TODO: Compare final backup tar with one from a trad tower
+- name: Write pg_dump to backup on PVC
+  community.kubernetes.k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ meta.name }}-db-management"
+    command: >-
+      bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db"
+  register: data_migration

roles/backup/tasks/secrets.yml

@@ -1,15 +1,10 @@
 ---
 
-# TODO: Get Secret_key value/s
-
-# TODO: Store Secret_key value/s in a way that can be made into another secret upon restore
-
-# The general idea here is that the user provides the name for the current deployment, we grab secrets based on that, then when it is restored, we restore to whatever name/namespace is specified at the time of restore
-
 - name: Make _secrets directory
   file:
     path: "_secrets"
     state: directory
+    mode: '0700'
 
 - name: Get secret_key
   k8s_info:
@@ -28,8 +23,9 @@
     dest: "_secrets/secret_key_secret.yml"
     mode: '0600'
 
-- set_fact:
-    secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml')}}"
+- name: Set secret key template
+  set_fact:
+    secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml') }}"
 
 - name: Write secret_key to pvc
   community.kubernetes.k8s_exec:
@@ -44,7 +40,6 @@
     namespace: '{{ meta.namespace }}'
     name: '{{ tower_admin_password_secret }}'
   register: _admin_password
-# TODO: check if admin_password secret name is provided, and check for that? use defaults.yml
 
 - name: Set admin_password
   set_fact:
@@ -56,8 +51,9 @@
     dest: "_secrets/admin_password_secret.yml"
     mode: '0600'
 
-- set_fact:
-    admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml')}}"
+- name: Set admin_password template
+  set_fact:
+    admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml') }}"
 
 - name: Write secret_key to pvc
   community.kubernetes.k8s_exec:
@@ -83,8 +79,9 @@
     dest: "_secrets/broadcast_websocket_secret.yml"
     mode: '0600'
 
-- set_fact:
-    broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml')}}"
+- name: Set broadcast_websocket template
+  set_fact:
+    broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml') }}"
 
 - name: Write secret_key to pvc
   community.kubernetes.k8s_exec:
@@ -115,8 +112,9 @@
     dest: "_secrets/postgres_secret.yml"
     mode: '0600'
 
-- set_fact:
-    postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml')}}"
+- name: Set postgres configuration
+  set_fact:
+    postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml') }}"
 
 - name: Write secret_key to pvc
   community.kubernetes.k8s_exec:

roles/backup/tasks/update_status.yml

@@ -0,0 +1,16 @@
+---
+- name: Set apiVersion and kind variables
+  set_fact:
+    api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
+    kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
+
+# The backup directory in this status can be referenced when restoring
+- name: Update Tower Backup status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ meta.name }}"
+    namespace: "{{ meta.namespace }}"
+    status:
+      towerBackupComplete: "{{ _backup_dir }}"
+  when: tower_backup_complete is defined

roles/backup/templates/awx_object.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: '{{ awx_api_version }}'
+kind: AWX
+metadata:
+{% raw %}
+  name: '{{ meta.name }}'
+  namespace: '{{ meta.namespace }}'
+{% endraw %}
+spec: {{ awx_spec }}

roles/installer/tasks/database_configuration.yml

@@ -110,6 +110,19 @@
     awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
     awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] |  default('prefer'|b64encode) | b64decode }}"
 
+# - name: Set apiVersion and kind variables
+#   set_fact:
+#     api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/")  }}'
+#     kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6]  }}'
+#
+# - name: Look up details for this deployment
+#   k8s_info:
+#     api_version: "{{ api_version }}"
+#     kind: "{{ kind }}"
+#     name: "{{ meta.name }}"
+#     namespace: "{{ meta.namespace }}"
+#   register: this_awx
+
 - name: Look up details for this deployment
   k8s_info:
     api_version: 'v1beta1'  # TODO: How to parameterize this?

roles/installer/tasks/update_status.yml

@@ -73,13 +73,3 @@
     status:
       towerMigratedFromSecret: "{{ tower_migrated_from_secret }}"
   when: tower_migrated_from_secret is defined
-
-- name: Update Tower Backup status
-  operator_sdk.util.k8s_status:
-    api_version: '{{ api_version }}'
-    kind: "{{ kind }}"
-    name: "{{ meta.name }}"
-    namespace: "{{ meta.namespace }}"
-    status:
-      towerBackupComplete: "{{ _backup_dir }}"
-  when: tower_backup_complete is defined

watches.yaml

@@ -9,6 +9,6 @@
 
 - version: v1beta1
   group: awx.ansible.com
-  kind: Backup
+  kind: AWXBackup
   role: backup
   reconcilePeriod: 360m