From 0580398c909337f867ea0105f69e868c7ff0166e Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 1 Apr 2021 22:57:37 -0400 Subject: [PATCH] Finish db restore logic - rename _backup_dir to backup_dir - add towerBackupClaim status to make the pvc name easier to find for users --- roles/backup/README.md | 2 +- roles/backup/tasks/awx-cro.yml | 2 +- roles/backup/tasks/main.yml | 2 +- roles/backup/tasks/postgres.yml | 10 ++-- roles/backup/tasks/secrets.yml | 8 +-- roles/backup/tasks/update_status.yml | 5 +- roles/backup/templates/event.yml.j2 | 4 +- roles/restore/README.md | 28 +++++++--- roles/restore/tasks/main.yml | 10 ++-- roles/restore/tasks/postgres.yml | 73 +++++++++++++-------------- roles/restore/tasks/update_status.yml | 4 +- roles/restore/templates/event.yml.j2 | 4 +- 12 files changed, 82 insertions(+), 70 deletions(-) diff --git a/roles/backup/README.md b/roles/backup/README.md index 8fa467d4..3d93035c 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -76,7 +76,7 @@ You can test this role directly by creating and running the following playbook w ``` --- -- name: Backup Tower +- name: Backup AWX hosts: localhost gather_facts: false roles: diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index a2b6df47..bcbb2bc5 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -35,4 +35,4 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ awx_object_template }}' > {{ _backup_dir }}/awx_object.yml" + bash -c "echo '{{ awx_object_template }}' > {{ backup_dir }}/awx_object.yml" diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index dfe9c05e..5fc6ab4d 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -24,7 +24,7 @@ - name: Set flag signifying this backup was successful set_fact: - tower_backup_complete: "{{ _backup_dir }}" + tower_backup_complete: true - include_tasks: cleanup.yml diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 2925be17..789c8d99 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -48,28 +48,28 @@ - name: Set backup directory name set_fact: - _backup_dir: "/backups/tower-openshift-backup-{{ now }}" + backup_dir: "/backups/tower-openshift-backup-{{ now }}" - name: Create directory for backup community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - mkdir -p {{ _backup_dir }} + mkdir -p {{ backup_dir }} - name: Precreate file for database dump community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - touch {{ _backup_dir }}/tower.db + touch {{ backup_dir }}/tower.db - name: Set permissions on file for database dump community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - chmod 0600 {{ _backup_dir }}/tower.db + chmod 0600 {{ backup_dir }}/tower.db - name: Set pg_dump command set_fact: @@ -85,5 +85,5 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" + bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ backup_dir }}/tower.db" register: data_migration diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index d5f0105e..5a276c0b 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -32,7 +32,7 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ secret_key_template }}' > {{ _backup_dir }}/secret_key_secret.yml" + bash -c "echo '{{ secret_key_template }}' > {{ backup_dir }}/secret_key_secret.yml" - name: Get admin_password k8s_info: @@ -60,7 +60,7 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ admin_password_template }}' > {{ _backup_dir }}/admin_password_secret.yml" + bash -c "echo '{{ admin_password_template }}' > {{ backup_dir }}/admin_password_secret.yml" - name: Get broadcast_websocket k8s_info: @@ -88,7 +88,7 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ broadcast_websocket_template }}' > {{ _backup_dir }}/broadcast_websocket_secret.yml" + bash -c "echo '{{ broadcast_websocket_template }}' > {{ backup_dir }}/broadcast_websocket_secret.yml" - name: Get postgres configuration k8s_info: @@ -121,4 +121,4 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ postgres_secret_template }}' > {{ _backup_dir }}/postgres_secret.yml" + bash -c "echo '{{ postgres_secret_template }}' > {{ backup_dir }}/postgres_secret.yml" diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml index 1c6ef1c6..b2894f22 100644 --- a/roles/backup/tasks/update_status.yml +++ b/roles/backup/tasks/update_status.yml @@ -12,5 +12,6 @@ name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" status: - towerBackupComplete: "{{ _backup_dir }}" - when: tower_backup_complete is defined + towerBackupDirectory: "{{ backup_dir }}" + towerBackupClaim: "{{ backup_pvc }}" + when: tower_backup_complete diff --git a/roles/backup/templates/event.yml.j2 b/roles/backup/templates/event.yml.j2 index 3670cba3..ead6aea4 100644 --- a/roles/backup/templates/event.yml.j2 +++ b/roles/backup/templates/event.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Event metadata: - name: restore-error.{{ now }} + name: backup-error.{{ now }} namespace: {{ meta.namespace }} involvedObject: apiVersion: awx.ansible.com/v1beta1 @@ -10,7 +10,7 @@ involvedObject: name: {{ meta.name }} namespace: {{ meta.namespace }} message: {{ error_msg }} -reason: RestoreFailed +reason: BackupFailed type: Warning firstTimestamp: {{ now }} lastTimestamp: {{ now }} diff --git a/roles/restore/README.md b/roles/restore/README.md index d04c4d83..16e1d7ce 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -25,7 +25,7 @@ Then create a file named `restore-awx.yml` with the following contents: apiVersion: awx.ansible.com/v1beta1 kind: AWXRestore metadata: - name: awxrestore + name: restore1 namespace: my-namespace spec: tower_name: mytower @@ -33,8 +33,7 @@ spec: tower_backup_dir: tower-openshift-backup-2021-04-01-15:49:17 ``` -Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. - +Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. Finally, use `kubectl` to create the restore object in your cluster: @@ -48,7 +47,24 @@ This will create a new deployment and restore your backup to it. Role Variables -------------- -A custom, pre-created pvc can be used by setting the following variables. +The name of the backup directory can be found as a status on your AWXBackup object. This can be found in your cluster's console, or with the client as shown below. + +```bash +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.towerBackupDirectory}" +/backups/tower-openshift-backup-2021-04-02-03:25:08 +``` + +``` +tower_backup_dir: '/backups/tower-openshift-backup-2021-04-02-03:25:08' +``` + + +The name of the PVC can also be found by looking at the backup object. + +```bash +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.towerBackupClaim}" +awx-backup-volume-claim +``` ``` tower_backup_pvc: 'awx-backup-volume-claim' @@ -69,11 +85,11 @@ You can test this role directly by creating and running the following playbook w ``` --- -- name: Backup Tower +- name: Restore AWX hosts: localhost gather_facts: false roles: - - backup + - restore ``` License diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index df03f3d1..8b072d73 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -20,13 +20,13 @@ - include_tasks: init.yml - # - include_tasks: postgres.yml - # - # - include_tasks: secrets.yml + - include_tasks: postgres.yml - - name: Set flag signifying this backup was successful + - include_tasks: secrets.yml + + - name: Set flag signifying this restore was successful set_fact: - tower_backup_complete: True + tower_restore_complete: True - include_tasks: cleanup.yml diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index e6d375e3..44acf750 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -42,44 +42,39 @@ set_fact: postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" +- name: Check for presence of AWX Deployment + k8s_info: + api_version: v1 + kind: Deployment + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + register: this_deployment -# TODO: Add postgres restore logic, just pipe the pg_dump from PVC to db via psql +- name: Scale down Deployment for migration + k8s_scale: + api_version: v1 + kind: Deployment + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + replicas: 0 + when: this_deployment['resources'] | length -# -# - name: Create directory for backup -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# mkdir -p {{ _backup_dir }} -# -# - name: Precreate file for database dump -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# touch {{ _backup_dir }}/tower.db -# -# - name: Set permissions on file for database dump -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# chmod 0600 {{ _backup_dir }}/tower.db -# -# - name: Set pg_dump command -# set_fact: -# pgdump: >- -# pg_dump --clean --create -# -h {{ awx_postgres_host }} -# -U {{ awx_postgres_user }} -# -d {{ awx_postgres_database }} -# -p {{ awx_postgres_port }} -# -# - name: Write pg_dump to backup on PVC -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" -# register: data_migration +- name: Set pg_restore command + set_fact: + psql_restore: >- + psql -U {{ awx_postgres_user }} + -d template1 + -p {{ awx_postgres_port }} + +- name: Restore database dump to the new postgresql container + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ postgres_pod_name }}" + command: | + bash -c """ + set -e -o pipefail + cat {{ tower_backup_dir }}/tower.db | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} + echo 'Successful' + """ + register: data_migration + failed_when: "'Successful' not in data_migration.stdout" diff --git a/roles/restore/tasks/update_status.yml b/roles/restore/tasks/update_status.yml index f056a1ac..92196d01 100644 --- a/roles/restore/tasks/update_status.yml +++ b/roles/restore/tasks/update_status.yml @@ -4,12 +4,12 @@ api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' -- name: Update Tower Backup status +- name: Update Tower Restore status operator_sdk.util.k8s_status: api_version: '{{ api_version }}' kind: "{{ kind }}" name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" status: - towerRestoreComplete: True + towerRestoreComplete: true when: tower_restore_complete is defined diff --git a/roles/restore/templates/event.yml.j2 b/roles/restore/templates/event.yml.j2 index ead6aea4..3670cba3 100644 --- a/roles/restore/templates/event.yml.j2 +++ b/roles/restore/templates/event.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Event metadata: - name: backup-error.{{ now }} + name: restore-error.{{ now }} namespace: {{ meta.namespace }} involvedObject: apiVersion: awx.ansible.com/v1beta1 @@ -10,7 +10,7 @@ involvedObject: name: {{ meta.name }} namespace: {{ meta.namespace }} message: {{ error_msg }} -reason: BackupFailed +reason: RestoreFailed type: Warning firstTimestamp: {{ now }} lastTimestamp: {{ now }}