From e1dca00f46843ae19ec5ae15a4aa158558236687 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 30 Mar 2021 16:57:01 -0400 Subject: [PATCH] Fix backup reconciliation loop, add error status --- .../crds/awx.ansible.com_awxbackups_crd.yaml | 28 ++-- .../awx.ansible.com_v1beta1_awxbackup_cr.yaml | 2 + roles/backup/README.md | 5 +- roles/backup/defaults/main.yml | 10 +- roles/backup/handlers/main.yml | 17 ++ roles/backup/store_secrets.yml | 7 - roles/backup/tasks/awx-cro.yml | 38 +++++ roles/backup/tasks/init.yml | 11 +- roles/backup/tasks/main.yml | 35 ++-- roles/backup/tasks/postgres.yml | 152 +++++++++--------- roles/backup/tasks/secrets.yml | 28 ++-- roles/backup/tasks/update_status.yml | 16 ++ roles/backup/templates/awx_object.yml.j2 | 9 ++ .../tasks/database_configuration.yml | 13 ++ roles/installer/tasks/update_status.yml | 10 -- watches.yaml | 2 +- 16 files changed, 243 insertions(+), 140 deletions(-) create mode 100644 roles/backup/handlers/main.yml delete mode 100644 roles/backup/store_secrets.yml create mode 100644 roles/backup/tasks/awx-cro.yml create mode 100644 roles/backup/tasks/update_status.yml create mode 100644 roles/backup/templates/awx_object.yml.j2 diff --git a/deploy/crds/awx.ansible.com_awxbackups_crd.yaml b/deploy/crds/awx.ansible.com_awxbackups_crd.yaml index f5303c49..5e9f9deb 100644 --- a/deploy/crds/awx.ansible.com_awxbackups_crd.yaml +++ b/deploy/crds/awx.ansible.com_awxbackups_crd.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -11,12 +12,21 @@ spec: singular: awxbackup scope: Namespaced versions: - - name: v1beta1 - schema: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true - served: true - storage: true - subresources: - status: {} + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + # TODO: Figure out how to require the tower_name field + properties: + spec: + type: object + properties: + tower_name: + description: Name of the deployment to be backed up + type: string diff --git a/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml b/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml index 7bd14a02..58ddf9eb 100644 --- a/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml +++ b/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml @@ -1,9 +1,11 @@ +--- apiVersion: awx.ansible.com/v1beta1 kind: AWXBackup metadata: name: example-awxbackup namespace: example-awx spec: + tower_name: '' tower_backup_pvc: '' tower_backup_size: '' tower_backup_storage_class: '' diff --git a/roles/backup/README.md b/roles/backup/README.md index 224a861d..8fa467d4 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -28,9 +28,12 @@ kind: AWXBackup metadata: name: awxbackup namespace: my-namespace +spec: + tower_name: mytower ``` -> The metadata.name you provide, is the name of the AWX deployment you intend to backup from (in case you have multiple in the same namespace). +Note that the `tower_name` above is the name of the AWX deployment you intend to backup from. + Finally, use `kubectl` to create the backup object in your cluster: diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 0c33617d..59310cdd 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,4 +1,6 @@ --- +# Required: specify name of tower deployment to backup from +tower_name: '' # Specify a pre-created PVC (name) to backup to tower_backup_pvc: '' @@ -10,7 +12,7 @@ tower_backup_size: '' tower_backup_storage_class: '' # Secret Names -tower_secret_key_secret: "{{ meta.name }}-secret-key" -tower_admin_password_secret: "{{ meta.name }}-admin-password" -tower_broadcast_websocket_secret: "{{ meta.name }}-broadcast-websocket" -tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration" +tower_secret_key_secret: "{{ tower_name }}-secret-key" +tower_admin_password_secret: "{{ tower_name }}-admin-password" +tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket" +tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration" diff --git a/roles/backup/handlers/main.yml b/roles/backup/handlers/main.yml new file mode 100644 index 00000000..e6ffb379 --- /dev/null +++ b/roles/backup/handlers/main.yml @@ -0,0 +1,17 @@ +--- + +- name: Update awxbackup status + block: + - name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + + - name: Update error status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + error: "{{ error_msg }}" diff --git a/roles/backup/store_secrets.yml b/roles/backup/store_secrets.yml deleted file mode 100644 index bd900004..00000000 --- a/roles/backup/store_secrets.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -- name: Test Getting and Storing Secrets (OCP) - hosts: localhost - - tasks: - - import_tasks: tasks/secrets.yml diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml new file mode 100644 index 00000000..a2b6df47 --- /dev/null +++ b/roles/backup/tasks/awx-cro.yml @@ -0,0 +1,38 @@ +--- + +- name: Get AWX custom resource object + k8s_info: + version: v1beta1 + kind: AWX + namespace: '{{ meta.namespace }}' + name: '{{ tower_name }}' + register: _awx_cro + +- name: Set AWX object + set_fact: + _awx: "{{ _awx_cro['resources'][0] }}" + +- name: Set apiVersion + set_fact: + awx_api_version: "{{ _awx['apiVersion'] }}" + +- name: Set user specified spec + set_fact: + awx_spec: "{{ _awx['spec'] }}" + +- name: Template AWX object definition + template: + src: awx_object.yml.j2 + dest: "_secrets/awx_object.yml" + mode: '0600' + +- name: Set AWX object template file as var + set_fact: + awx_object_template: "{{ lookup('file', '_secrets/awx_object.yml') }}" + +- name: Write awx object to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ awx_object_template }}' > {{ _backup_dir }}/awx_object.yml" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 87d4cf46..85c056fd 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -19,6 +19,11 @@ when: - tower_backup_pvc != '' or tower_backup_pvc is defined +- name: Update status + set_fact: + error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + notify: "Update awxbackup status" + - name: Fail early if pvc is defined but does not exist fail: msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." @@ -29,12 +34,10 @@ set_fact: _default_backup_pvc: "{{ meta.name }}-backup-claim" +# by default, it will re-use the old pvc if already created (unless pvc is provided) - name: Set PVC to use for backup set_fact: - backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true)}}" - -# TODO: re-use the old pvc if already created (unless pvc is provided) -# TODO: allow users to configure their own storage class for dynamically creating a pvc? + backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" - name: Create PVC for backup community.kubernetes.k8s: diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 458cf9b7..dfe9c05e 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,26 +1,37 @@ --- +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + - name: Look up details for this deployment k8s_info: - api_version: 'v1beta1' # TODO: How to parameterize this? - kind: "AWX" # TODO: How to parameterize this? + api_version: "{{ api_version }}" + kind: "{{ kind }}" name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" - register: this_awx + register: this_backup - block: - - include_tasks: init.yml + - include_tasks: init.yml - - include_tasks: postgres.yml + - include_tasks: postgres.yml - - include_tasks: secrets.yml + - include_tasks: secrets.yml - # TODO: Add task to change the status on the backup CR when this runs successfully - - name: Set flag signifying this backup was successful - set_fact: - tower_backup_complete: "{{ _backup_dir }}" + - include_tasks: awx-cro.yml - - include_tasks: cleanup.yml + - name: Set flag signifying this backup was successful + set_fact: + tower_backup_complete: "{{ _backup_dir }}" + + - include_tasks: cleanup.yml when: - - this_awx['resources'][0]['status']['towerMigratedFromSecret'] is not defined + - this_backup['resources'][0]['status']['towerBackupComplete'] is not defined + +- name: Update status variables + include_tasks: update_status.yml + +# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object. See ticket diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index f17987b8..2925be17 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -1,91 +1,89 @@ --- - - name: Check for specified PostgreSQL configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ tower_postgres_configuration_secret }}' - register: _custom_pg_config_resources - when: tower_postgres_configuration_secret | length - - name: Check for default PostgreSQL configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ meta.name }}-postgres-configuration' - register: _default_pg_config_resources +- name: Check for specified PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_postgres_configuration_secret }}' + register: _custom_pg_config_resources + when: tower_postgres_configuration_secret | length - - name: Set PostgreSQL configuration - set_fact: - pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' +- name: Check for default PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_name }}-postgres-configuration' + register: _default_pg_config_resources - - name: Store Database Configuration - set_fact: - awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" - awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" - awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" - awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" - awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" +- name: Set PostgreSQL configuration + set_fact: + pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' - - name: Get the postgres pod information - k8s_info: - kind: Pod - namespace: '{{ meta.namespace }}' - label_selectors: - - "app={{ meta.name }}-{{ deployment_type }}-postgres" - register: postgres_pod - until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" - delay: 5 - retries: 60 +- name: Store Database Configuration + set_fact: + awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" - - name: Set the resource pod name as a variable. - set_fact: - postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "app={{ tower_name }}-{{ deployment_type }}-postgres" + register: postgres_pod + until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 - - name: Determine the timestamp for the backup once for all nodes - set_fact: - now: '{{ lookup("pipe", "date +%F-%T") }}' +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" - - name: Set backup directory name - set_fact: - _backup_dir: "/backups/tower-openshift-backup-{{ now }}" +- name: Determine the timestamp for the backup once for all nodes + set_fact: + now: '{{ lookup("pipe", "date +%F-%T") }}' - - name: Create directory for backup - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - mkdir -p {{ _backup_dir }} +- name: Set backup directory name + set_fact: + _backup_dir: "/backups/tower-openshift-backup-{{ now }}" - - name: Precreate file for database dump - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - touch {{ _backup_dir }}/tower.db +- name: Create directory for backup + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + mkdir -p {{ _backup_dir }} - - name: Set permissions on file for database dump - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - chmod 0600 {{ _backup_dir }}/tower.db +- name: Precreate file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + touch {{ _backup_dir }}/tower.db - - name: Set pg_dump command - set_fact: - pgdump: >- - pg_dump --clean --create - -h {{ awx_postgres_host }} - -U {{ awx_postgres_user }} - -d {{ awx_postgres_database }} - -p {{ awx_postgres_port }} +- name: Set permissions on file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + chmod 0600 {{ _backup_dir }}/tower.db - - name: Write pg_dump to backup on PVC - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" - register: data_migration +- name: Set pg_dump command + set_fact: + pgdump: >- + pg_dump --clean --create + -h {{ awx_postgres_host }} + -U {{ awx_postgres_user }} + -d {{ awx_postgres_database }} + -p {{ awx_postgres_port }} - # TODO: Backup secret key and other secrets - look at trad tower backup pattern - # TODO: Compare final backup tar with one from a trad tower +- name: Write pg_dump to backup on PVC + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" + register: data_migration diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index ee30e690..cb062190 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -1,15 +1,10 @@ --- -# TODO: Get Secret_key value/s - -# TODO: Store Secret_key value/s in a way that can be made into another secret upon restore - -# The general idea here is that the user provides the name for the current deployment, we grab secrets based on that, then when it is restored, we restore to whatever name/namespace is specified at the time of restore - - name: Make _secrets directory file: path: "_secrets" state: directory + mode: '0700' - name: Get secret_key k8s_info: @@ -28,8 +23,9 @@ dest: "_secrets/secret_key_secret.yml" mode: '0600' -- set_fact: - secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml')}}" +- name: Set secret key template + set_fact: + secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: @@ -44,7 +40,6 @@ namespace: '{{ meta.namespace }}' name: '{{ tower_admin_password_secret }}' register: _admin_password -# TODO: check if admin_password secret name is provided, and check for that? use defaults.yml - name: Set admin_password set_fact: @@ -56,8 +51,9 @@ dest: "_secrets/admin_password_secret.yml" mode: '0600' -- set_fact: - admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml')}}" +- name: Set admin_password template + set_fact: + admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: @@ -83,8 +79,9 @@ dest: "_secrets/broadcast_websocket_secret.yml" mode: '0600' -- set_fact: - broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml')}}" +- name: Set broadcast_websocket template + set_fact: + broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: @@ -115,8 +112,9 @@ dest: "_secrets/postgres_secret.yml" mode: '0600' -- set_fact: - postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml')}}" +- name: Set postgres configuration + set_fact: + postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml new file mode 100644 index 00000000..1c6ef1c6 --- /dev/null +++ b/roles/backup/tasks/update_status.yml @@ -0,0 +1,16 @@ +--- +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +# The backup directory in this status can be referenced when restoring +- name: Update Tower Backup status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerBackupComplete: "{{ _backup_dir }}" + when: tower_backup_complete is defined diff --git a/roles/backup/templates/awx_object.yml.j2 b/roles/backup/templates/awx_object.yml.j2 new file mode 100644 index 00000000..b0a539d9 --- /dev/null +++ b/roles/backup/templates/awx_object.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: '{{ awx_api_version }}' +kind: AWX +metadata: +{% raw %} + name: '{{ meta.name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +spec: {{ awx_spec }} diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index ec476f7c..18d408fd 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -110,6 +110,19 @@ awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}" +# - name: Set apiVersion and kind variables +# set_fact: +# api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' +# kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' +# +# - name: Look up details for this deployment +# k8s_info: +# api_version: "{{ api_version }}" +# kind: "{{ kind }}" +# name: "{{ meta.name }}" +# namespace: "{{ meta.namespace }}" +# register: this_awx + - name: Look up details for this deployment k8s_info: api_version: 'v1beta1' # TODO: How to parameterize this? diff --git a/roles/installer/tasks/update_status.yml b/roles/installer/tasks/update_status.yml index 93068f1c..ec4b3d54 100644 --- a/roles/installer/tasks/update_status.yml +++ b/roles/installer/tasks/update_status.yml @@ -73,13 +73,3 @@ status: towerMigratedFromSecret: "{{ tower_migrated_from_secret }}" when: tower_migrated_from_secret is defined - -- name: Update Tower Backup status - operator_sdk.util.k8s_status: - api_version: '{{ api_version }}' - kind: "{{ kind }}" - name: "{{ meta.name }}" - namespace: "{{ meta.namespace }}" - status: - towerBackupComplete: "{{ _backup_dir }}" - when: tower_backup_complete is defined diff --git a/watches.yaml b/watches.yaml index 01114c09..3d59e606 100644 --- a/watches.yaml +++ b/watches.yaml @@ -9,6 +9,6 @@ - version: v1beta1 group: awx.ansible.com - kind: Backup + kind: AWXBackup role: backup reconcilePeriod: 360m