From 0220c7588447432fe3601e76379190e1f591f375 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 17 Mar 2021 12:26:14 -0400 Subject: [PATCH 01/43] wip deployment podspec or sts --- roles/backup/tasks/main.yml | 150 +++++++++++++++++++ roles/backup/templates/backup_pvc.yml.j2 | 12 ++ roles/backup/templates/management-pod.yml.j2 | 0 3 files changed, 162 insertions(+) create mode 100644 roles/backup/tasks/main.yml create mode 100644 roles/backup/templates/backup_pvc.yml.j2 create mode 100644 roles/backup/templates/management-pod.yml.j2 diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml new file mode 100644 index 00000000..1a0223eb --- /dev/null +++ b/roles/backup/tasks/main.yml @@ -0,0 +1,150 @@ +--- +# - name: Check for old PostgreSQL configuration secret +# k8s_info: +# kind: Secret +# namespace: '{{ meta.namespace }}' +# name: '{{ tower_postgres_configuration_secret }}' +# register: old_pg_config +# +# - name: Migrate data from old Openshift instance +# import_tasks: migrate_data.yml +# when: old_pg_config['resources'][0]['data']['host'] is defined +# ignore_errors: true + + + +# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +# Break up database_configuration.yml and just import those tasks here + +- name: Check for specified PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_postgres_configuration_secret }}' + register: _custom_pg_config_resources + when: tower_postgres_configuration_secret | length + +- name: Check for default PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ meta.name }}-postgres-configuration' + register: _default_pg_config_resources + +- name: Set PostgreSQL configuration + set_fact: + _pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' + +- block: + - name: Create Database configuration + k8s: + apply: true + definition: "{{ lookup('template', 'tower_postgres_secret.yaml.j2') }}" + + - name: Read Database Configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ meta.name }}-postgres-configuration' + register: _generated_pg_config_resources + when: not _pg_config['resources'] | default([]) | length + +- name: Set PostgreSQL Configuration + set_fact: + pg_config: '{{ _generated_pg_config_resources["resources"] | default([]) | length | ternary(_generated_pg_config_resources, _pg_config) }}' + +- name: Store Database Configuration + set_fact: + awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + +# ///////////////// + +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "app={{ deployment_type }}-postgres" + register: postgres_pod + until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 + +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" + +- name: Determine the timestamp for the backup once for all nodes + set_fact: + now: '{{ lookup("pipe", "date +%F-%T") }}' + + + +### define a volumeClaimTemplate in the management-pod.yml.j2 + +- name: Delete any existing management pod + shell: | + {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \ + delete pod ansible-tower-management --grace-period=0 --ignore-not-found + +- name: Template management pod + set_fact: + management_pod: "{{ lookup('template', 'management-pod.yml.j2') }}" + +- name: Create management pod + shell: | + echo {{ management_pod | quote }} | {{ kubectl_or_oc }} apply -f - + +- name: Wait for management pod to start + shell: | + {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \ + get pod ansible-tower-management -o jsonpath="{.status.phase}" + register: result + until: result.stdout == "Running" + retries: 60 + delay: 10 + + + + + + + + + +- name: Check for existing PVC + +- name: Create PVC for backup + +- name: Create PVC to backup to if no PVC exists + k8s: + apply: true + definition: "{{ lookup('template', 'backup_pvc.yaml.j2') }}" + # when: # pvc doesn't exist already + # - pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' +# tower_backup_pvc: '' +# tower_backup_size: '' + + +- name: Set pg_dump command + set_fact: + pgdump: >- + pg_dump --clean --create + -h {{ tower_postgres_host }} + -U {{ tower_postgres_user }} + -d {{ tower_postgres_database }} + -p {{ tower_postgres_port }} + + +- name: Stream backup from pg_dump to the new postgresql container + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ postgres_pod_name }}" + command: >- + bash -c "PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} > {{ playbook_dir }}/tower-openshift-backup-{{ now }}/tower.db" + ignore_errors: true + register: data_migration diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 new file mode 100644 index 00000000..e0905272 --- /dev/null +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: "v1" +kind: "PersistentVolumeClaim" +metadata: + name: "{{ tower_backup_pvc | default('tower_backup') }}_{{ now }}" +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "5Gi" + volumeName: "tower_backup_{{ now }}" diff --git a/roles/backup/templates/management-pod.yml.j2 b/roles/backup/templates/management-pod.yml.j2 new file mode 100644 index 00000000..e69de29b From e037feafbfe570421b93baa96b1306d466bdd9f7 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 19 Mar 2021 16:45:26 -0400 Subject: [PATCH 02/43] Create management pod and pvc for backup --- roles/backup/defaults/main.yml | 18 ++ roles/backup/tasks/main.yml | 179 ++++++++++--------- roles/backup/templates/backup_pv.yml.j2 | 16 ++ roles/backup/templates/backup_pvc.yml.j2 | 13 +- roles/backup/templates/management-pod.yml.j2 | 20 +++ 5 files changed, 156 insertions(+), 90 deletions(-) create mode 100644 roles/backup/defaults/main.yml create mode 100644 roles/backup/templates/backup_pv.yml.j2 diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml new file mode 100644 index 00000000..9ebff0c5 --- /dev/null +++ b/roles/backup/defaults/main.yml @@ -0,0 +1,18 @@ +--- +deployment_type: awx + +# Secret to lookup that provide the secret key +# +tower_secret_key_secret: '' + +tower_postgres_storage_class: '' +tower_postgres_data_path: '/var/lib/postgresql/data/pgdata' + +# Secret to lookup that provide the PostgreSQL configuration +# +tower_postgres_configuration_secret: '' +tower_old_postgres_configuration_secret: '' + +tower_postgres_image: postgres:12 +tower_backup_pvc: '' +tower_backup_size: '' diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 1a0223eb..27f6581e 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,20 +1,4 @@ --- -# - name: Check for old PostgreSQL configuration secret -# k8s_info: -# kind: Secret -# namespace: '{{ meta.namespace }}' -# name: '{{ tower_postgres_configuration_secret }}' -# register: old_pg_config -# -# - name: Migrate data from old Openshift instance -# import_tasks: migrate_data.yml -# when: old_pg_config['resources'][0]['data']['host'] is defined -# ignore_errors: true - - - -# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\ -# Break up database_configuration.yml and just import those tasks here - name: Check for specified PostgreSQL configuration k8s_info: @@ -33,25 +17,7 @@ - name: Set PostgreSQL configuration set_fact: - _pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' - -- block: - - name: Create Database configuration - k8s: - apply: true - definition: "{{ lookup('template', 'tower_postgres_secret.yaml.j2') }}" - - - name: Read Database Configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ meta.name }}-postgres-configuration' - register: _generated_pg_config_resources - when: not _pg_config['resources'] | default([]) | length - -- name: Set PostgreSQL Configuration - set_fact: - pg_config: '{{ _generated_pg_config_resources["resources"] | default([]) | length | ternary(_generated_pg_config_resources, _pg_config) }}' + pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' - name: Store Database Configuration set_fact: @@ -61,8 +27,6 @@ awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" -# ///////////////// - - name: Get the postgres pod information k8s_info: kind: Pod @@ -83,68 +47,115 @@ now: '{{ lookup("pipe", "date +%F-%T") }}' +# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. +- name: Check provided PVC exists + k8s_info: + name: "{{ tower_backup_pvc }}" + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + register: provided_pvc + when: + - tower_backup_pvc != '' or tower_backup_pvc is defined -### define a volumeClaimTemplate in the management-pod.yml.j2 +# or should we automatically create a PVC for them with this name if it doesn't exist? +- name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + when: provided_pvc.resources | length == 0 + +# If tower_backup_pvc is defined, use in management-pod.yml.j2 +- name: Set default pvc name # to get around nested jinja2 vars + set_fact: + _default_backup_pvc: "{{ deployment_type }}-backup-pvc" + +- name: Set PVC to use for backup + set_fact: + backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true)}}" + +# TODO: handle re-using existing pv and pvc, or make new onces with auto-generated name? +- block: + - name: Create PV for backup + community.kubernetes.k8s: + name: "{{ deployment_type }}-backup-pv" + kind: PersistentVolume + namespace: "{{ meta.namespace }}" + template: "backup_pv.yml.j2" + + - name: Create PVC for backup + community.kubernetes.k8s: + name: "{{ deployment_type }}-backup-pvc" + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + template: "backup_pvc.yml.j2" + when: + - tower_backup_pvc == '' or tower_backup_pvc is not defined - name: Delete any existing management pod - shell: | - {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \ - delete pod ansible-tower-management --grace-period=0 --ignore-not-found + community.kubernetes.k8s: + name: "{{ deployment_type }}-db-management" + kind: Deployment + namespace: "{{ meta.namespace }}" + state: absent + force: true -- name: Template management pod +- name: Create management pod from templated deployment config + community.kubernetes.k8s: + name: "{{ deployment_type }}-db-management" + kind: Deployment + namespace: "{{ meta.namespace }}" + state: present + template: "management-pod.yml.j2" + wait: true + +- name: Set backup directory name set_fact: - management_pod: "{{ lookup('template', 'management-pod.yml.j2') }}" + _backup_dir: "/backups/tower-openshift-backup-{{ now }}" -- name: Create management pod - shell: | - echo {{ management_pod | quote }} | {{ kubectl_or_oc }} apply -f - +- name: Create directory for backup + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ deployment_type }}-db-management" + command: >- + mkdir -p {{ _backup_dir }} -- name: Wait for management pod to start - shell: | - {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \ - get pod ansible-tower-management -o jsonpath="{.status.phase}" - register: result - until: result.stdout == "Running" - retries: 60 - delay: 10 - - - - - - - - - -- name: Check for existing PVC - -- name: Create PVC for backup - -- name: Create PVC to backup to if no PVC exists - k8s: - apply: true - definition: "{{ lookup('template', 'backup_pvc.yaml.j2') }}" - # when: # pvc doesn't exist already - # - pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' -# tower_backup_pvc: '' -# tower_backup_size: '' +- name: Precreate file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ deployment_type }}-db-management" + command: >- + touch {{ _backup_dir }}/tower.db +- name: Set permissions on file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ deployment_type }}-db-management" + command: >- + chmod 0600 {{ _backup_dir }}/tower.db + chown postgres:postgres {{ _backup_dir }}/tower.db - name: Set pg_dump command set_fact: pgdump: >- pg_dump --clean --create - -h {{ tower_postgres_host }} - -U {{ tower_postgres_user }} - -d {{ tower_postgres_database }} - -p {{ tower_postgres_port }} + -h {{ awx_postgres_host }} + -U {{ awx_postgres_user }} + -d {{ awx_postgres_database }} + -p {{ awx_postgres_port }} - -- name: Stream backup from pg_dump to the new postgresql container +- name: Write pg_dump to backup on PVC community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" - pod: "{{ postgres_pod_name }}" + pod: "{{ deployment_type }}-db-management" command: >- - bash -c "PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} > {{ playbook_dir }}/tower-openshift-backup-{{ now }}/tower.db" - ignore_errors: true + bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" register: data_migration + +# Backup secret key and other secrets - look at trad tower backup pattern + +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ deployment_type }}-db-management" + kind: Deployment + namespace: "{{ meta.namespace }}" + state: absent + force: true diff --git a/roles/backup/templates/backup_pv.yml.j2 b/roles/backup/templates/backup_pv.yml.j2 new file mode 100644 index 00000000..cabc217a --- /dev/null +++ b/roles/backup/templates/backup_pv.yml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ deployment_type }}-backup-pv + namespace: {{ meta.namespace}} + labels: + type: awx-backup +spec: + storageClassName: standard + capacity: + storage: "{{ tower_backup_size | default('5Gi', true) }}" + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/data" diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index e0905272..1b3bbc2a 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -1,12 +1,13 @@ --- -apiVersion: "v1" -kind: "PersistentVolumeClaim" +apiVersion: v1 +kind: PersistentVolumeClaim metadata: - name: "{{ tower_backup_pvc | default('tower_backup') }}_{{ now }}" + name: "{{ deployment_type }}-backup-pvc" spec: accessModes: - - "ReadWriteOnce" + - ReadWriteOnce + storageClassName: standard resources: requests: - storage: "5Gi" - volumeName: "tower_backup_{{ now }}" + storage: "{{ tower_backup_size | default('5Gi', true) }}" + volumeName: "{{ deployment_type }}-backup-pv" diff --git a/roles/backup/templates/management-pod.yml.j2 b/roles/backup/templates/management-pod.yml.j2 index e69de29b..0c914c0a 100644 --- a/roles/backup/templates/management-pod.yml.j2 +++ b/roles/backup/templates/management-pod.yml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ deployment_type }}-db-management + namespace: {{ meta.namespace }} +spec: + containers: + - name: {{ deployment_type }}-db-management + image: "{{ tower_postgres_image }}" + imagePullPolicy: Always + command: ["sleep", "infinity"] + volumeMounts: + - name: "{{ deployment_type }}-backup" + mountPath: /backups + volumes: + - name: "{{ deployment_type }}-backup" + persistentVolumeClaim: + claimName: "{{ backup_pvc }}" + restartPolicy: Never From 4a5ca184c094819ea8fdae99376a9ee8f0671c0e Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 23 Mar 2021 01:40:25 -0400 Subject: [PATCH 03/43] Use storage class to dynamically create volume for backups --- roles/backup/defaults/main.yml | 2 +- roles/backup/tasks/main.yml | 49 ++++++++------------ roles/backup/templates/backup_pv.yml.j2 | 16 ------- roles/backup/templates/backup_pvc.yml.j2 | 6 +-- roles/backup/templates/management-pod.yml.j2 | 8 ++-- 5 files changed, 29 insertions(+), 52 deletions(-) delete mode 100644 roles/backup/templates/backup_pv.yml.j2 diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 9ebff0c5..92ad117c 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,5 +1,5 @@ --- -deployment_type: awx +deployment_type: "{{ meta.name | default('awx', true)}}" # Secret to lookup that provide the secret key # diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 27f6581e..7b52fd28 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -46,6 +46,14 @@ set_fact: now: '{{ lookup("pipe", "date +%F-%T") }}' +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ deployment_type }}-db-management" + kind: Pod + namespace: "{{ meta.namespace }}" + state: absent + force: true + wait: true # Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. - name: Check provided PVC exists @@ -57,47 +65,31 @@ when: - tower_backup_pvc != '' or tower_backup_pvc is defined -# or should we automatically create a PVC for them with this name if it doesn't exist? - name: Fail early if pvc is defined but does not exist fail: msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." when: provided_pvc.resources | length == 0 # If tower_backup_pvc is defined, use in management-pod.yml.j2 -- name: Set default pvc name # to get around nested jinja2 vars +- name: Set default pvc name set_fact: - _default_backup_pvc: "{{ deployment_type }}-backup-pvc" + _default_backup_pvc: "{{ deployment_type }}-backup-claim" - name: Set PVC to use for backup set_fact: backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true)}}" -# TODO: handle re-using existing pv and pvc, or make new onces with auto-generated name? -- block: - - name: Create PV for backup - community.kubernetes.k8s: - name: "{{ deployment_type }}-backup-pv" - kind: PersistentVolume - namespace: "{{ meta.namespace }}" - template: "backup_pv.yml.j2" +# TODO: re-use the old pvc if already created (unless pvc is provided) +# TODO: allow users to configure their own storage class for dynamically creating a pvc? - - name: Create PVC for backup - community.kubernetes.k8s: - name: "{{ deployment_type }}-backup-pvc" - kind: PersistentVolumeClaim - namespace: "{{ meta.namespace }}" - template: "backup_pvc.yml.j2" +- name: Create PVC for backup + community.kubernetes.k8s: + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + template: "backup_pvc.yml.j2" when: - tower_backup_pvc == '' or tower_backup_pvc is not defined -- name: Delete any existing management pod - community.kubernetes.k8s: - name: "{{ deployment_type }}-db-management" - kind: Deployment - namespace: "{{ meta.namespace }}" - state: absent - force: true - - name: Create management pod from templated deployment config community.kubernetes.k8s: name: "{{ deployment_type }}-db-management" @@ -131,10 +123,9 @@ pod: "{{ deployment_type }}-db-management" command: >- chmod 0600 {{ _backup_dir }}/tower.db - chown postgres:postgres {{ _backup_dir }}/tower.db - name: Set pg_dump command - set_fact: + set_fact: pgdump: >- pg_dump --clean --create -h {{ awx_postgres_host }} @@ -150,12 +141,12 @@ bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" register: data_migration -# Backup secret key and other secrets - look at trad tower backup pattern +# TODO: Backup secret key and other secrets - look at trad tower backup pattern - name: Delete any existing management pod community.kubernetes.k8s: name: "{{ deployment_type }}-db-management" - kind: Deployment + kind: Pod namespace: "{{ meta.namespace }}" state: absent force: true diff --git a/roles/backup/templates/backup_pv.yml.j2 b/roles/backup/templates/backup_pv.yml.j2 deleted file mode 100644 index cabc217a..00000000 --- a/roles/backup/templates/backup_pv.yml.j2 +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: {{ deployment_type }}-backup-pv - namespace: {{ meta.namespace}} - labels: - type: awx-backup -spec: - storageClassName: standard - capacity: - storage: "{{ tower_backup_size | default('5Gi', true) }}" - accessModes: - - ReadWriteOnce - hostPath: - path: "/mnt/data" diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index 1b3bbc2a..14891d28 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -2,12 +2,12 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: "{{ deployment_type }}-backup-pvc" + name: {{ deployment_type }}-backup-claim + namespace: {{ meta.namespace}} spec: accessModes: - ReadWriteOnce storageClassName: standard resources: requests: - storage: "{{ tower_backup_size | default('5Gi', true) }}" - volumeName: "{{ deployment_type }}-backup-pv" + storage: {{ tower_backup_size | default('5Gi', true) }} diff --git a/roles/backup/templates/management-pod.yml.j2 b/roles/backup/templates/management-pod.yml.j2 index 0c914c0a..57b08f1f 100644 --- a/roles/backup/templates/management-pod.yml.j2 +++ b/roles/backup/templates/management-pod.yml.j2 @@ -11,10 +11,12 @@ spec: imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: - - name: "{{ deployment_type }}-backup" + - name: {{ deployment_type }}-backup mountPath: /backups + readOnly: false volumes: - - name: "{{ deployment_type }}-backup" + - name: {{ deployment_type }}-backup persistentVolumeClaim: - claimName: "{{ backup_pvc }}" + claimName: {{ backup_pvc }} + readOnly: false restartPolicy: Never From fdcc745f111bc77ef729eafb6b14ddda2f7c0315 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 23 Mar 2021 11:19:25 -0400 Subject: [PATCH 04/43] Add watcher for backup CR --- roles/backup/tasks/main.yml | 1 + watches.yaml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 7b52fd28..fcbf876c 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -142,6 +142,7 @@ register: data_migration # TODO: Backup secret key and other secrets - look at trad tower backup pattern +# TODO: Compare final backup tar with one from a trad tower - name: Delete any existing management pod community.kubernetes.k8s: diff --git a/watches.yaml b/watches.yaml index 02e2eb99..824c161f 100644 --- a/watches.yaml +++ b/watches.yaml @@ -6,3 +6,9 @@ finalizer: name: finalizer.awx.ansible.com role: finalizer + +- version: v1beta1 + group: awx.ansible.com + kind: AWXBackup + role: /opt/ansible/roles/backup + reconcilePeriod: 360m From bcd141043822055a54017a9bd76cee25e6dfb68a Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 23 Mar 2021 15:33:03 -0400 Subject: [PATCH 05/43] init backup CR files --- deploy/awx-operator.yaml | 150 ++++++++++++++++++ deploy/crds/awx.ansible.com_backups_crd.yaml | 22 +++ .../awx.ansible.com_v1beta1_backup_cr.yaml | 10 ++ roles/backup/README.md | 81 ++++++++++ roles/backup/defaults/main.yml | 15 -- roles/backup/meta/main.yml | 30 ++++ roles/backup/templates/backup_pvc.yml.j2 | 2 +- roles/backup/vars/main.yml | 13 ++ watches.yaml | 11 +- 9 files changed, 315 insertions(+), 19 deletions(-) create mode 100644 deploy/crds/awx.ansible.com_backups_crd.yaml create mode 100644 deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml create mode 100644 roles/backup/README.md create mode 100644 roles/backup/meta/main.yml create mode 100644 roles/backup/vars/main.yml diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 2cd87de5..f87b157d 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -1,5 +1,155 @@ # This file is generated by Ansible. Changes will be lost. # Update templates under ansible/templates/ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: awx-operator +rules: + - apiGroups: + - route.openshift.io + resources: + - routes + - routes/custom-host + verbs: + - '*' + - apiGroups: + - "" + - "rbac.authorization.k8s.io" + resources: + - pods + - services + - services/finalizers + - serviceaccounts + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + - roles + - rolebindings + verbs: + - '*' + - apiGroups: + - apps + - extensions + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + - ingresses + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - apps + resourceNames: + - awx-operator + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - apps + resources: + - deployments/scale + verbs: + - patch + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - apiGroups: + - awx.ansible.com + resources: + - '*' + - backups + verbs: + - '*' + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: awx-operator +subjects: + - kind: ServiceAccount + name: awx-operator + namespace: default +roleRef: + kind: ClusterRole + name: awx-operator + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: awx-operator + namespace: default + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: awx-operator +spec: + replicas: 1 + selector: + matchLabels: + name: awx-operator + template: + metadata: + labels: + name: awx-operator + spec: + serviceAccountName: awx-operator + containers: + - name: awx-operator + image: "quay.io/ansible/awx-operator:0.7.0" + imagePullPolicy: "Always" + volumeMounts: + - mountPath: /tmp/ansible-operator/runner + name: runner + env: + # Watch all namespaces (cluster-scoped). + - name: WATCH_NAMESPACE + value: "" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: awx-operator + - name: ANSIBLE_GATHERING + value: explicit + livenessProbe: + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - name: runner + emptyDir: {} + --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/deploy/crds/awx.ansible.com_backups_crd.yaml b/deploy/crds/awx.ansible.com_backups_crd.yaml new file mode 100644 index 00000000..03063186 --- /dev/null +++ b/deploy/crds/awx.ansible.com_backups_crd.yaml @@ -0,0 +1,22 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: backups.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: Backup + listKind: BackupList + plural: backups + singular: backup + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + served: true + storage: true + subresources: + status: {} diff --git a/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml b/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml new file mode 100644 index 00000000..505c0c9d --- /dev/null +++ b/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml @@ -0,0 +1,10 @@ +apiVersion: awx.ansible.com/v1beta1 +kind: Backup +metadata: + name: example-awx + namespace: example-awx +spec: + tower_backup_pvc: '' + tower_backup_size: '' + tower_postgres_storage_class: '' + tower_postgres_configuration_secret: '' diff --git a/roles/backup/README.md b/roles/backup/README.md new file mode 100644 index 00000000..e3f35116 --- /dev/null +++ b/roles/backup/README.md @@ -0,0 +1,81 @@ +Role Name +========= + +The purpose of this role is to create a backup of your AWX deployment. This includes: + - backup of the postgresql database + - secret_key + - custom user config files + - manual projects + + +Requirements +------------ + +This role assumes you are authenticated with an Openshift or Kubernetes cluster which: + - The awx-operator has been deployed to + - AWX is deployed to via the operator + + +Usage +---------------- + +Then create a file named `backup-awx.yml` with the following contents: + +```yaml +--- +apiVersion: awx.ansible.com/v1beta1 +kind: Backup +metadata: + name: awx + namespace: my-namespace +``` + +Finally, use `kubectl` to create the awx instance in your cluster: + +```bash +#> kubectl apply -f backup-awx.yml +``` + + + +Role Variables +-------------- + +A custom, pre-created pvc can be used by setting the following variables. + +``` +tower_backup_pvc: 'awx-backup-volume-claim' +``` + +This role will automatically create a pvc using a Storage Class if provided: + +``` +tower_postgres_storage_class: 'standard' +tower_backup_size: '20Gi' +``` + +If a custom postgres configuration secret was used when deploying AWX, it must be set: + +``` +tower_postgres_configuration_secret: 'awx-postgres-configuration' +``` + + +Testing +---------------- + +You can test this role directly by creating and running the following playbook with the appropriate variables: + +``` +--- +- name: Backup Tower + hosts: localhost + gather_facts: false + roles: + - backup +``` + +License +------- + +MIT diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 92ad117c..a0809604 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,18 +1,3 @@ --- deployment_type: "{{ meta.name | default('awx', true)}}" - -# Secret to lookup that provide the secret key -# -tower_secret_key_secret: '' - -tower_postgres_storage_class: '' -tower_postgres_data_path: '/var/lib/postgresql/data/pgdata' - -# Secret to lookup that provide the PostgreSQL configuration -# -tower_postgres_configuration_secret: '' -tower_old_postgres_configuration_secret: '' - tower_postgres_image: postgres:12 -tower_backup_pvc: '' -tower_backup_size: '' diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml new file mode 100644 index 00000000..c48a9cf2 --- /dev/null +++ b/roles/backup/meta/main.yml @@ -0,0 +1,30 @@ +--- +galaxy_info: + author: Ansible + description: AWX role for AWX Operator for Kubernetes. + company: Red Hat, Inc. + + license: MIT + + min_ansible_version: 2.8 + + platforms: + - name: EL + versions: + - all + - name: Debian + versions: + - all + + galaxy_tags: + - tower + - awx + - ansible + - backup + - automation + +dependencies: [] + +collections: + - community.kubernetes + - operator_sdk.util diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index 14891d28..ca1411f0 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -7,7 +7,7 @@ metadata: spec: accessModes: - ReadWriteOnce - storageClassName: standard + storageClassName: {{ tower_postgres_storage_class }} resources: requests: storage: {{ tower_backup_size | default('5Gi', true) }} diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml new file mode 100644 index 00000000..5f653505 --- /dev/null +++ b/roles/backup/vars/main.yml @@ -0,0 +1,13 @@ +--- + +# Specify a pre-created PVC (name) to backup to +tower_backup_pvc: '' + +# Size of backup PVC if created dynamically +tower_backup_size: '' + +# Specify storage class to determine how to dynamically create PVC's with +tower_postgres_storage_class: '' + +# Secret to lookup that provide the PostgreSQL configuration +tower_postgres_configuration_secret: '' diff --git a/watches.yaml b/watches.yaml index 824c161f..3b7034f5 100644 --- a/watches.yaml +++ b/watches.yaml @@ -7,8 +7,13 @@ name: finalizer.awx.ansible.com role: finalizer +# - version: v1beta1 +# group: awx.ansible.com +# kind: Backup +# role: /opt/ansible/roles/backup +# reconcilePeriod: 360m + - version: v1beta1 group: awx.ansible.com - kind: AWXBackup - role: /opt/ansible/roles/backup - reconcilePeriod: 360m + kind: Backup + role: backup From 54efda1a25bf69e6313603819e9c99ea957a09ab Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 23 Mar 2021 17:15:15 -0400 Subject: [PATCH 06/43] Use default cluster storage class if none is provided --- roles/backup/README.md | 3 +++ roles/backup/templates/backup_pvc.yml.j2 | 2 ++ 2 files changed, 5 insertions(+) diff --git a/roles/backup/README.md b/roles/backup/README.md index e3f35116..f5fe33a4 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -36,6 +36,7 @@ Finally, use `kubectl` to create the awx instance in your cluster: #> kubectl apply -f backup-awx.yml ``` +The resulting pvc will contain a backup tar that can be used to restore to a new deployment. Future backups will also be stored in separate tars on the same pvc. Role Variables @@ -47,6 +48,8 @@ A custom, pre-created pvc can be used by setting the following variables. tower_backup_pvc: 'awx-backup-volume-claim' ``` +> If no pvc or storage class is provided, the cluster's default storage class will be used to create the pvc. + This role will automatically create a pvc using a Storage Class if provided: ``` diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index ca1411f0..3f918dda 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -7,7 +7,9 @@ metadata: spec: accessModes: - ReadWriteOnce +{% if tower_postgres_storage_class != '' %} storageClassName: {{ tower_postgres_storage_class }} +{% endif %} resources: requests: storage: {{ tower_backup_size | default('5Gi', true) }} From 9e44e21a66b374bf4c5247733226064e6c1f5540 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 24 Mar 2021 11:27:08 -0400 Subject: [PATCH 07/43] Rename pvc name var to be consistent with other backup variables --- deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml | 2 +- roles/backup/README.md | 4 ++-- roles/backup/templates/backup_pvc.yml.j2 | 4 ++-- roles/backup/vars/main.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml b/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml index 505c0c9d..e026600b 100644 --- a/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml +++ b/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml @@ -6,5 +6,5 @@ metadata: spec: tower_backup_pvc: '' tower_backup_size: '' - tower_postgres_storage_class: '' + tower_backup_storage_class: '' tower_postgres_configuration_secret: '' diff --git a/roles/backup/README.md b/roles/backup/README.md index f5fe33a4..98f42547 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -30,7 +30,7 @@ metadata: namespace: my-namespace ``` -Finally, use `kubectl` to create the awx instance in your cluster: +Finally, use `kubectl` to create the backup object in your cluster: ```bash #> kubectl apply -f backup-awx.yml @@ -53,7 +53,7 @@ tower_backup_pvc: 'awx-backup-volume-claim' This role will automatically create a pvc using a Storage Class if provided: ``` -tower_postgres_storage_class: 'standard' +tower_backup_storage_class: 'standard' tower_backup_size: '20Gi' ``` diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index 3f918dda..6c997de8 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -7,8 +7,8 @@ metadata: spec: accessModes: - ReadWriteOnce -{% if tower_postgres_storage_class != '' %} - storageClassName: {{ tower_postgres_storage_class }} +{% if tower_backup_storage_class != '' %} + storageClassName: {{ tower_backup_storage_class }} {% endif %} resources: requests: diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index 5f653505..7ee44aa6 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -7,7 +7,7 @@ tower_backup_pvc: '' tower_backup_size: '' # Specify storage class to determine how to dynamically create PVC's with -tower_postgres_storage_class: '' +tower_backup_storage_class: '' # Secret to lookup that provide the PostgreSQL configuration tower_postgres_configuration_secret: '' From 13397f41add0282435f03dd34119793c051b6844 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 24 Mar 2021 13:23:41 -0400 Subject: [PATCH 08/43] use meta.data to keep pods and pvcs unique in the same namespace --- roles/backup/README.md | 2 ++ roles/backup/defaults/main.yml | 2 +- roles/backup/tasks/main.yml | 28 +++++++++++++------- roles/backup/templates/backup_pvc.yml.j2 | 2 +- roles/backup/templates/management-pod.yml.j2 | 8 +++--- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/roles/backup/README.md b/roles/backup/README.md index 98f42547..8d4a3ef9 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -30,6 +30,8 @@ metadata: namespace: my-namespace ``` +> The metadata.name you provide, is the name of the AWX deployment you intend to backup from (in case you have multiple in the same namespace). + Finally, use `kubectl` to create the backup object in your cluster: ```bash diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index a0809604..dbb48bf8 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,3 +1,3 @@ --- -deployment_type: "{{ meta.name | default('awx', true)}}" +deployment_type: "awx" tower_postgres_image: postgres:12 diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index fcbf876c..7ad04265 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,5 +1,15 @@ --- +- include_tasks: init.yml + +- include_tasks: postgres.yml + +- include_tasks: projects.yml + +- include_tasks: conf.yml + +- include_tasks: download.yml + - name: Check for specified PostgreSQL configuration k8s_info: kind: Secret @@ -32,7 +42,7 @@ kind: Pod namespace: '{{ meta.namespace }}' label_selectors: - - "app={{ deployment_type }}-postgres" + - "app={{ meta.name }}-postgres" register: postgres_pod until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" delay: 5 @@ -48,7 +58,7 @@ - name: Delete any existing management pod community.kubernetes.k8s: - name: "{{ deployment_type }}-db-management" + name: "{{ meta.name }}-db-management" kind: Pod namespace: "{{ meta.namespace }}" state: absent @@ -73,7 +83,7 @@ # If tower_backup_pvc is defined, use in management-pod.yml.j2 - name: Set default pvc name set_fact: - _default_backup_pvc: "{{ deployment_type }}-backup-claim" + _default_backup_pvc: "{{ meta.name }}-backup-claim" - name: Set PVC to use for backup set_fact: @@ -92,7 +102,7 @@ - name: Create management pod from templated deployment config community.kubernetes.k8s: - name: "{{ deployment_type }}-db-management" + name: "{{ meta.name }}-db-management" kind: Deployment namespace: "{{ meta.namespace }}" state: present @@ -106,21 +116,21 @@ - name: Create directory for backup community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" - pod: "{{ deployment_type }}-db-management" + pod: "{{ meta.name }}-db-management" command: >- mkdir -p {{ _backup_dir }} - name: Precreate file for database dump community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" - pod: "{{ deployment_type }}-db-management" + pod: "{{ meta.name }}-db-management" command: >- touch {{ _backup_dir }}/tower.db - name: Set permissions on file for database dump community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" - pod: "{{ deployment_type }}-db-management" + pod: "{{ meta.name }}-db-management" command: >- chmod 0600 {{ _backup_dir }}/tower.db @@ -136,7 +146,7 @@ - name: Write pg_dump to backup on PVC community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" - pod: "{{ deployment_type }}-db-management" + pod: "{{ meta.name }}-db-management" command: >- bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" register: data_migration @@ -146,7 +156,7 @@ - name: Delete any existing management pod community.kubernetes.k8s: - name: "{{ deployment_type }}-db-management" + name: "{{ meta.name }}-db-management" kind: Pod namespace: "{{ meta.namespace }}" state: absent diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index 6c997de8..693b00ec 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ deployment_type }}-backup-claim + name: {{ meta.name }}-backup-claim namespace: {{ meta.namespace}} spec: accessModes: diff --git a/roles/backup/templates/management-pod.yml.j2 b/roles/backup/templates/management-pod.yml.j2 index 57b08f1f..87ddff8e 100644 --- a/roles/backup/templates/management-pod.yml.j2 +++ b/roles/backup/templates/management-pod.yml.j2 @@ -2,20 +2,20 @@ apiVersion: v1 kind: Pod metadata: - name: {{ deployment_type }}-db-management + name: {{ meta.name }}-db-management namespace: {{ meta.namespace }} spec: containers: - - name: {{ deployment_type }}-db-management + - name: {{ meta.name }}-db-management image: "{{ tower_postgres_image }}" imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: - - name: {{ deployment_type }}-backup + - name: {{ meta.name }}-backup mountPath: /backups readOnly: false volumes: - - name: {{ deployment_type }}-backup + - name: {{ meta.name }}-backup persistentVolumeClaim: claimName: {{ backup_pvc }} readOnly: false From 0a82fec359d295b28099b64f32ca928cf8bf182a Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 25 Mar 2021 01:31:18 -0400 Subject: [PATCH 09/43] Refactor backup role & store secrets as well --- roles/backup/defaults/main.yml | 5 + roles/backup/store_secrets.yml | 7 + roles/backup/tasks/cleanup.yml | 9 + roles/backup/tasks/init.yml | 54 ++++++ roles/backup/tasks/main.yml | 165 +------------------ roles/backup/tasks/postgres.yml | 91 ++++++++++ roles/backup/tasks/secrets.yml | 59 +++++++ roles/backup/templates/admin_password.yml.j2 | 10 ++ roles/backup/templates/secret_key.yml.j2 | 10 ++ 9 files changed, 253 insertions(+), 157 deletions(-) create mode 100644 roles/backup/store_secrets.yml create mode 100644 roles/backup/tasks/cleanup.yml create mode 100644 roles/backup/tasks/init.yml create mode 100644 roles/backup/tasks/postgres.yml create mode 100644 roles/backup/tasks/secrets.yml create mode 100644 roles/backup/templates/admin_password.yml.j2 create mode 100644 roles/backup/templates/secret_key.yml.j2 diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index dbb48bf8..046fd48a 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,3 +1,8 @@ --- deployment_type: "awx" tower_postgres_image: postgres:12 + +# Secret Names +tower_secret_key_secret: "{{ meta.name }}-secret-key" +tower_admin_password_secret: "{{ meta.name }}-admin-password" +# tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration" diff --git a/roles/backup/store_secrets.yml b/roles/backup/store_secrets.yml new file mode 100644 index 00000000..bd900004 --- /dev/null +++ b/roles/backup/store_secrets.yml @@ -0,0 +1,7 @@ +--- + +- name: Test Getting and Storing Secrets (OCP) + hosts: localhost + + tasks: + - import_tasks: tasks/secrets.yml diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml new file mode 100644 index 00000000..9976a8c9 --- /dev/null +++ b/roles/backup/tasks/cleanup.yml @@ -0,0 +1,9 @@ +--- + +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ meta.namespace }}" + state: absent + force: true diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml new file mode 100644 index 00000000..87d4cf46 --- /dev/null +++ b/roles/backup/tasks/init.yml @@ -0,0 +1,54 @@ +--- + +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ meta.namespace }}" + state: absent + force: true + wait: true + +# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. +- name: Check provided PVC exists + k8s_info: + name: "{{ tower_backup_pvc }}" + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + register: provided_pvc + when: + - tower_backup_pvc != '' or tower_backup_pvc is defined + +- name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + when: provided_pvc.resources | length == 0 + +# If tower_backup_pvc is defined, use in management-pod.yml.j2 +- name: Set default pvc name + set_fact: + _default_backup_pvc: "{{ meta.name }}-backup-claim" + +- name: Set PVC to use for backup + set_fact: + backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true)}}" + +# TODO: re-use the old pvc if already created (unless pvc is provided) +# TODO: allow users to configure their own storage class for dynamically creating a pvc? + +- name: Create PVC for backup + community.kubernetes.k8s: + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + template: "backup_pvc.yml.j2" + when: + - tower_backup_pvc == '' or tower_backup_pvc is not defined + +- name: Create management pod from templated deployment config + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Deployment + namespace: "{{ meta.namespace }}" + state: present + template: "management-pod.yml.j2" + wait: true diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 7ad04265..52852290 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,163 +1,14 @@ --- -- include_tasks: init.yml +# - include_tasks: init.yml -- include_tasks: postgres.yml +- include_tasks: secrets.yml -- include_tasks: projects.yml +# - include_tasks: postgres.yml -- include_tasks: conf.yml +# +## - include_tasks: conf.yml +# +## - include_tasks: download.yml -- include_tasks: download.yml - -- name: Check for specified PostgreSQL configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ tower_postgres_configuration_secret }}' - register: _custom_pg_config_resources - when: tower_postgres_configuration_secret | length - -- name: Check for default PostgreSQL configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ meta.name }}-postgres-configuration' - register: _default_pg_config_resources - -- name: Set PostgreSQL configuration - set_fact: - pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' - -- name: Store Database Configuration - set_fact: - awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" - awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" - awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" - awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" - awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" - -- name: Get the postgres pod information - k8s_info: - kind: Pod - namespace: '{{ meta.namespace }}' - label_selectors: - - "app={{ meta.name }}-postgres" - register: postgres_pod - until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" - delay: 5 - retries: 60 - -- name: Set the resource pod name as a variable. - set_fact: - postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" - -- name: Determine the timestamp for the backup once for all nodes - set_fact: - now: '{{ lookup("pipe", "date +%F-%T") }}' - -- name: Delete any existing management pod - community.kubernetes.k8s: - name: "{{ meta.name }}-db-management" - kind: Pod - namespace: "{{ meta.namespace }}" - state: absent - force: true - wait: true - -# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. -- name: Check provided PVC exists - k8s_info: - name: "{{ tower_backup_pvc }}" - kind: PersistentVolumeClaim - namespace: "{{ meta.namespace }}" - register: provided_pvc - when: - - tower_backup_pvc != '' or tower_backup_pvc is defined - -- name: Fail early if pvc is defined but does not exist - fail: - msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." - when: provided_pvc.resources | length == 0 - -# If tower_backup_pvc is defined, use in management-pod.yml.j2 -- name: Set default pvc name - set_fact: - _default_backup_pvc: "{{ meta.name }}-backup-claim" - -- name: Set PVC to use for backup - set_fact: - backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true)}}" - -# TODO: re-use the old pvc if already created (unless pvc is provided) -# TODO: allow users to configure their own storage class for dynamically creating a pvc? - -- name: Create PVC for backup - community.kubernetes.k8s: - kind: PersistentVolumeClaim - namespace: "{{ meta.namespace }}" - template: "backup_pvc.yml.j2" - when: - - tower_backup_pvc == '' or tower_backup_pvc is not defined - -- name: Create management pod from templated deployment config - community.kubernetes.k8s: - name: "{{ meta.name }}-db-management" - kind: Deployment - namespace: "{{ meta.namespace }}" - state: present - template: "management-pod.yml.j2" - wait: true - -- name: Set backup directory name - set_fact: - _backup_dir: "/backups/tower-openshift-backup-{{ now }}" - -- name: Create directory for backup - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - mkdir -p {{ _backup_dir }} - -- name: Precreate file for database dump - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - touch {{ _backup_dir }}/tower.db - -- name: Set permissions on file for database dump - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - chmod 0600 {{ _backup_dir }}/tower.db - -- name: Set pg_dump command - set_fact: - pgdump: >- - pg_dump --clean --create - -h {{ awx_postgres_host }} - -U {{ awx_postgres_user }} - -d {{ awx_postgres_database }} - -p {{ awx_postgres_port }} - -- name: Write pg_dump to backup on PVC - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" - register: data_migration - -# TODO: Backup secret key and other secrets - look at trad tower backup pattern -# TODO: Compare final backup tar with one from a trad tower - -- name: Delete any existing management pod - community.kubernetes.k8s: - name: "{{ meta.name }}-db-management" - kind: Pod - namespace: "{{ meta.namespace }}" - state: absent - force: true +- include_tasks: cleanup.yml diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml new file mode 100644 index 00000000..f17987b8 --- /dev/null +++ b/roles/backup/tasks/postgres.yml @@ -0,0 +1,91 @@ +--- + - name: Check for specified PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_postgres_configuration_secret }}' + register: _custom_pg_config_resources + when: tower_postgres_configuration_secret | length + + - name: Check for default PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ meta.name }}-postgres-configuration' + register: _default_pg_config_resources + + - name: Set PostgreSQL configuration + set_fact: + pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' + + - name: Store Database Configuration + set_fact: + awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + + - name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "app={{ meta.name }}-{{ deployment_type }}-postgres" + register: postgres_pod + until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 + + - name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" + + - name: Determine the timestamp for the backup once for all nodes + set_fact: + now: '{{ lookup("pipe", "date +%F-%T") }}' + + - name: Set backup directory name + set_fact: + _backup_dir: "/backups/tower-openshift-backup-{{ now }}" + + - name: Create directory for backup + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + mkdir -p {{ _backup_dir }} + + - name: Precreate file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + touch {{ _backup_dir }}/tower.db + + - name: Set permissions on file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + chmod 0600 {{ _backup_dir }}/tower.db + + - name: Set pg_dump command + set_fact: + pgdump: >- + pg_dump --clean --create + -h {{ awx_postgres_host }} + -U {{ awx_postgres_user }} + -d {{ awx_postgres_database }} + -p {{ awx_postgres_port }} + + - name: Write pg_dump to backup on PVC + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" + register: data_migration + + # TODO: Backup secret key and other secrets - look at trad tower backup pattern + # TODO: Compare final backup tar with one from a trad tower diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml new file mode 100644 index 00000000..98506dbc --- /dev/null +++ b/roles/backup/tasks/secrets.yml @@ -0,0 +1,59 @@ +--- + +# TODO: Get Secret_key value/s + +# TODO: Store Secret_key value/s in a way that can be made into another secret upon restore + +# The general idea here is that the user provides the name for the current deployment, we grab secrets based on that, then when it is restored, we restore to whatever name/namespace is specified at the time of restore + +- name: Make _secrets directory + file: + path: "{{ playbook_dir }}/_secrets" + state: directory + +- name: Get secret_key + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_secret_key_secret }}' + register: _secret_key + +- name: Set secret key + set_fact: + secret_key: "{{ _secret_key['resources'][0]['data']['secret_key'] | b64decode }}" + +- name: Template secret_key definition + template: + src: secret_key.yml.j2 + dest: "{{ playbook_dir }}/_secrets/secrets.yml" + mode: '0600' + # dest: pvc # potentially just do a copy task, loop through definition files + +- name: Get admin_password + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_admin_password_secret }}' + register: _admin_password +# TODO: check if admin_password secret name is provided, and check for that? use defaults.yml + +- name: Set admin_password + set_fact: + admin_password: "{{ _admin_password['resources'][0]['data']['password'] | b64decode }}" + +- name: Template admin_password definition + template: + src: admin_password.yml.j2 + dest: "{{ playbook_dir }}/_secrets/admin_password.yml" + mode: '0600' + + +# TODO: Secrets to back up: tower-secret-key, tower1-admin-password, tower1-app-credentials, tower1-broadcast-websocket, tower1-dockercfg-q8qd2, tower1-postgres-configuration +# Do we need the service-account-token? probably? `tower1-token-hn2hm`, tower1-token-slllw + + +# After copying secret files to the PVC, delete the local tmp copies +- name: Clean up _secrets directory + ansible.builtin.file: + path: "{{ playbook_dir }}/_secrets" + state: absent diff --git a/roles/backup/templates/admin_password.yml.j2 b/roles/backup/templates/admin_password.yml.j2 new file mode 100644 index 00000000..3d1df46e --- /dev/null +++ b/roles/backup/templates/admin_password.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ meta.name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + password: '{{ admin_password }}' diff --git a/roles/backup/templates/secret_key.yml.j2 b/roles/backup/templates/secret_key.yml.j2 new file mode 100644 index 00000000..febc23ee --- /dev/null +++ b/roles/backup/templates/secret_key.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ meta.name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + secret_key: '{{ secret_key }}' From 91dda5cb16e2e0327b4a17b60abee21d365c75d8 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 26 Mar 2021 13:42:33 -0400 Subject: [PATCH 10/43] backup secrets to YAML files --- ansible/templates/role.yml.j2 | 1 + roles/backup/defaults/main.yml | 5 - roles/backup/tasks/cleanup.yml | 7 ++ roles/backup/tasks/main.yml | 13 +-- roles/backup/tasks/secrets.yml | 93 ++++++++++++++++--- ...rd.yml.j2 => admin_password_secret.yml.j2} | 0 .../broadcast_websocket_secret.yml.j2 | 10 ++ roles/backup/templates/postgres_secret.yml.j2 | 16 ++++ ...et_key.yml.j2 => secret_key_secret.yml.j2} | 0 roles/backup/vars/main.yml | 7 +- roles/installer/tasks/update_status.yml | 10 ++ 11 files changed, 136 insertions(+), 26 deletions(-) rename roles/backup/templates/{admin_password.yml.j2 => admin_password_secret.yml.j2} (100%) create mode 100644 roles/backup/templates/broadcast_websocket_secret.yml.j2 create mode 100644 roles/backup/templates/postgres_secret.yml.j2 rename roles/backup/templates/{secret_key.yml.j2 => secret_key_secret.yml.j2} (100%) diff --git a/ansible/templates/role.yml.j2 b/ansible/templates/role.yml.j2 index 1b1263b3..ce13ebc3 100644 --- a/ansible/templates/role.yml.j2 +++ b/ansible/templates/role.yml.j2 @@ -79,5 +79,6 @@ rules: - awx.ansible.com resources: - '*' + - backups verbs: - '*' diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 046fd48a..dbb48bf8 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,8 +1,3 @@ --- deployment_type: "awx" tower_postgres_image: postgres:12 - -# Secret Names -tower_secret_key_secret: "{{ meta.name }}-secret-key" -tower_admin_password_secret: "{{ meta.name }}-admin-password" -# tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration" diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml index 9976a8c9..e20e8718 100644 --- a/roles/backup/tasks/cleanup.yml +++ b/roles/backup/tasks/cleanup.yml @@ -1,5 +1,12 @@ --- +# After copying secret files to the PVC, delete the local tmp copies +- name: Clean up _secrets directory + ansible.builtin.file: + path: "{{ playbook_dir }}/_secrets" + state: absent + + - name: Delete any existing management pod community.kubernetes.k8s: name: "{{ meta.name }}-db-management" diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 52852290..2dcfb230 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,14 +1,15 @@ --- -# - include_tasks: init.yml +- include_tasks: init.yml + +- include_tasks: postgres.yml - include_tasks: secrets.yml -# - include_tasks: postgres.yml +# TODO: Add task to change the status on the backup CR when this runs successfully +- name: Set flag signifying this backup was successful + set_fact: + tower_backup_complete: "{{ _backup_dir }}" -# -## - include_tasks: conf.yml -# -## - include_tasks: download.yml - include_tasks: cleanup.yml diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 98506dbc..ee30e690 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -8,7 +8,7 @@ - name: Make _secrets directory file: - path: "{{ playbook_dir }}/_secrets" + path: "_secrets" state: directory - name: Get secret_key @@ -24,10 +24,19 @@ - name: Template secret_key definition template: - src: secret_key.yml.j2 - dest: "{{ playbook_dir }}/_secrets/secrets.yml" + src: secret_key_secret.yml.j2 + dest: "_secrets/secret_key_secret.yml" mode: '0600' - # dest: pvc # potentially just do a copy task, loop through definition files + +- set_fact: + secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml')}}" + +- name: Write secret_key to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ secret_key_template }}' > {{ _backup_dir }}/secret_key_secret.yml" - name: Get admin_password k8s_info: @@ -43,17 +52,75 @@ - name: Template admin_password definition template: - src: admin_password.yml.j2 - dest: "{{ playbook_dir }}/_secrets/admin_password.yml" + src: admin_password_secret.yml.j2 + dest: "_secrets/admin_password_secret.yml" mode: '0600' +- set_fact: + admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml')}}" -# TODO: Secrets to back up: tower-secret-key, tower1-admin-password, tower1-app-credentials, tower1-broadcast-websocket, tower1-dockercfg-q8qd2, tower1-postgres-configuration -# Do we need the service-account-token? probably? `tower1-token-hn2hm`, tower1-token-slllw +- name: Write secret_key to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ admin_password_template }}' > {{ _backup_dir }}/admin_password_secret.yml" +- name: Get broadcast_websocket + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_broadcast_websocket_secret }}' + register: _broadcast_websocket -# After copying secret files to the PVC, delete the local tmp copies -- name: Clean up _secrets directory - ansible.builtin.file: - path: "{{ playbook_dir }}/_secrets" - state: absent +- name: Set broadcast_websocket key + set_fact: + secret_key: "{{ _broadcast_websocket['resources'][0]['data']['secret'] | b64decode }}" + +- name: Template broadcast_websocket definition + template: + src: broadcast_websocket_secret.yml.j2 + dest: "_secrets/broadcast_websocket_secret.yml" + mode: '0600' + +- set_fact: + broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml')}}" + +- name: Write secret_key to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ broadcast_websocket_template }}' > {{ _backup_dir }}/broadcast_websocket_secret.yml" + +- name: Get postgres configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_postgres_configuration_secret }}' + register: _postgres_configuration + +- name: Set postgres configuration + set_fact: + database_password: "{{ _postgres_configuration['resources'][0]['data']['password'] | b64decode }}" + database_username: "{{ _postgres_configuration['resources'][0]['data']['username'] | b64decode }}" + database_name: "{{ _postgres_configuration['resources'][0]['data']['database'] | b64decode }}" + database_port: "{{ _postgres_configuration['resources'][0]['data']['port'] | b64decode }}" + database_host: "{{ _postgres_configuration['resources'][0]['data']['host'] | b64decode }}" + database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode }}" + +- name: Template postgres configuration definition + template: + src: postgres_secret.yml.j2 + dest: "_secrets/postgres_secret.yml" + mode: '0600' + +- set_fact: + postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml')}}" + +- name: Write secret_key to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ postgres_secret_template }}' > {{ _backup_dir }}/postgres_secret.yml" diff --git a/roles/backup/templates/admin_password.yml.j2 b/roles/backup/templates/admin_password_secret.yml.j2 similarity index 100% rename from roles/backup/templates/admin_password.yml.j2 rename to roles/backup/templates/admin_password_secret.yml.j2 diff --git a/roles/backup/templates/broadcast_websocket_secret.yml.j2 b/roles/backup/templates/broadcast_websocket_secret.yml.j2 new file mode 100644 index 00000000..c8d4cc2f --- /dev/null +++ b/roles/backup/templates/broadcast_websocket_secret.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ meta.name }}-broadcast-websocket' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + secret: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}' diff --git a/roles/backup/templates/postgres_secret.yml.j2 b/roles/backup/templates/postgres_secret.yml.j2 new file mode 100644 index 00000000..b1ad1f6b --- /dev/null +++ b/roles/backup/templates/postgres_secret.yml.j2 @@ -0,0 +1,16 @@ +# Postgres Secret. +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ meta.name }}-postgres-configuration' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + password: '{{ database_password }}' + username: '{{ database_username }}' + database: '{{ database_name }}' + port: '{{ database_port }}' + host: '{{ database_host }}' + type: '{{ database_type }}' diff --git a/roles/backup/templates/secret_key.yml.j2 b/roles/backup/templates/secret_key_secret.yml.j2 similarity index 100% rename from roles/backup/templates/secret_key.yml.j2 rename to roles/backup/templates/secret_key_secret.yml.j2 diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index 7ee44aa6..0c33617d 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -9,5 +9,8 @@ tower_backup_size: '' # Specify storage class to determine how to dynamically create PVC's with tower_backup_storage_class: '' -# Secret to lookup that provide the PostgreSQL configuration -tower_postgres_configuration_secret: '' +# Secret Names +tower_secret_key_secret: "{{ meta.name }}-secret-key" +tower_admin_password_secret: "{{ meta.name }}-admin-password" +tower_broadcast_websocket_secret: "{{ meta.name }}-broadcast-websocket" +tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration" diff --git a/roles/installer/tasks/update_status.yml b/roles/installer/tasks/update_status.yml index ec4b3d54..93068f1c 100644 --- a/roles/installer/tasks/update_status.yml +++ b/roles/installer/tasks/update_status.yml @@ -73,3 +73,13 @@ status: towerMigratedFromSecret: "{{ tower_migrated_from_secret }}" when: tower_migrated_from_secret is defined + +- name: Update Tower Backup status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerBackupComplete: "{{ _backup_dir }}" + when: tower_backup_complete is defined From 4839bdcaad0ae14f50313bbbafe42e4ad22386e5 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 26 Mar 2021 16:40:36 -0400 Subject: [PATCH 11/43] Rename Backup CR to AWXBackup to be more unique - we could alternatively direct users to use the full GVK. Issue is potential conflict with AH operator CRs --- deploy/crds/awx.ansible.com_backups_crd.yaml | 10 +++---- .../awx.ansible.com_v1beta1_backup_cr.yaml | 4 +-- roles/backup/README.md | 4 +-- roles/backup/tasks/main.yml | 27 +++++++++++++------ watches.yaml | 7 +---- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/deploy/crds/awx.ansible.com_backups_crd.yaml b/deploy/crds/awx.ansible.com_backups_crd.yaml index 03063186..f5303c49 100644 --- a/deploy/crds/awx.ansible.com_backups_crd.yaml +++ b/deploy/crds/awx.ansible.com_backups_crd.yaml @@ -1,14 +1,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: backups.awx.ansible.com + name: awxbackups.awx.ansible.com spec: group: awx.ansible.com names: - kind: Backup - listKind: BackupList - plural: backups - singular: backup + kind: AWXBackup + listKind: AWXBackupList + plural: awxbackups + singular: awxbackup scope: Namespaced versions: - name: v1beta1 diff --git a/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml b/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml index e026600b..7bd14a02 100644 --- a/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml +++ b/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml @@ -1,7 +1,7 @@ apiVersion: awx.ansible.com/v1beta1 -kind: Backup +kind: AWXBackup metadata: - name: example-awx + name: example-awxbackup namespace: example-awx spec: tower_backup_pvc: '' diff --git a/roles/backup/README.md b/roles/backup/README.md index 8d4a3ef9..224a861d 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -24,9 +24,9 @@ Then create a file named `backup-awx.yml` with the following contents: ```yaml --- apiVersion: awx.ansible.com/v1beta1 -kind: Backup +kind: AWXBackup metadata: - name: awx + name: awxbackup namespace: my-namespace ``` diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 2dcfb230..458cf9b7 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,15 +1,26 @@ --- -- include_tasks: init.yml +- name: Look up details for this deployment + k8s_info: + api_version: 'v1beta1' # TODO: How to parameterize this? + kind: "AWX" # TODO: How to parameterize this? + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + register: this_awx -- include_tasks: postgres.yml +- block: + - include_tasks: init.yml -- include_tasks: secrets.yml + - include_tasks: postgres.yml -# TODO: Add task to change the status on the backup CR when this runs successfully -- name: Set flag signifying this backup was successful - set_fact: - tower_backup_complete: "{{ _backup_dir }}" + - include_tasks: secrets.yml + # TODO: Add task to change the status on the backup CR when this runs successfully + - name: Set flag signifying this backup was successful + set_fact: + tower_backup_complete: "{{ _backup_dir }}" -- include_tasks: cleanup.yml + - include_tasks: cleanup.yml + + when: + - this_awx['resources'][0]['status']['towerMigratedFromSecret'] is not defined diff --git a/watches.yaml b/watches.yaml index 3b7034f5..01114c09 100644 --- a/watches.yaml +++ b/watches.yaml @@ -7,13 +7,8 @@ name: finalizer.awx.ansible.com role: finalizer -# - version: v1beta1 -# group: awx.ansible.com -# kind: Backup -# role: /opt/ansible/roles/backup -# reconcilePeriod: 360m - - version: v1beta1 group: awx.ansible.com kind: Backup role: backup + reconcilePeriod: 360m From f17dcdc3e9f9c9877801864949ce87e206aada0d Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Mon, 29 Mar 2021 11:00:04 -0400 Subject: [PATCH 12/43] Swap vars and defaults, rename to awxbackups --- ansible/templates/role.yml.j2 | 2 +- ...yaml => awx.ansible.com_awxbackups_crd.yaml} | 0 ...> awx.ansible.com_v1beta1_awxbackup_cr.yaml} | 0 roles/backup/defaults/main.yml | 17 +++++++++++++++-- roles/backup/vars/main.yml | 16 ++-------------- 5 files changed, 18 insertions(+), 17 deletions(-) rename deploy/crds/{awx.ansible.com_backups_crd.yaml => awx.ansible.com_awxbackups_crd.yaml} (100%) rename deploy/crds/{awx.ansible.com_v1beta1_backup_cr.yaml => awx.ansible.com_v1beta1_awxbackup_cr.yaml} (100%) diff --git a/ansible/templates/role.yml.j2 b/ansible/templates/role.yml.j2 index ce13ebc3..80405470 100644 --- a/ansible/templates/role.yml.j2 +++ b/ansible/templates/role.yml.j2 @@ -79,6 +79,6 @@ rules: - awx.ansible.com resources: - '*' - - backups + - awxbackups verbs: - '*' diff --git a/deploy/crds/awx.ansible.com_backups_crd.yaml b/deploy/crds/awx.ansible.com_awxbackups_crd.yaml similarity index 100% rename from deploy/crds/awx.ansible.com_backups_crd.yaml rename to deploy/crds/awx.ansible.com_awxbackups_crd.yaml diff --git a/deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml b/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml similarity index 100% rename from deploy/crds/awx.ansible.com_v1beta1_backup_cr.yaml rename to deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index dbb48bf8..0c33617d 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,3 +1,16 @@ --- -deployment_type: "awx" -tower_postgres_image: postgres:12 + +# Specify a pre-created PVC (name) to backup to +tower_backup_pvc: '' + +# Size of backup PVC if created dynamically +tower_backup_size: '' + +# Specify storage class to determine how to dynamically create PVC's with +tower_backup_storage_class: '' + +# Secret Names +tower_secret_key_secret: "{{ meta.name }}-secret-key" +tower_admin_password_secret: "{{ meta.name }}-admin-password" +tower_broadcast_websocket_secret: "{{ meta.name }}-broadcast-websocket" +tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration" diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index 0c33617d..851b98f0 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -1,16 +1,4 @@ --- -# Specify a pre-created PVC (name) to backup to -tower_backup_pvc: '' - -# Size of backup PVC if created dynamically -tower_backup_size: '' - -# Specify storage class to determine how to dynamically create PVC's with -tower_backup_storage_class: '' - -# Secret Names -tower_secret_key_secret: "{{ meta.name }}-secret-key" -tower_admin_password_secret: "{{ meta.name }}-admin-password" -tower_broadcast_websocket_secret: "{{ meta.name }}-broadcast-websocket" -tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration" +deployment_type: "awx" +tower_postgres_image: postgres:12 From e1dca00f46843ae19ec5ae15a4aa158558236687 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 30 Mar 2021 16:57:01 -0400 Subject: [PATCH 13/43] Fix backup reconciliation loop, add error status --- .../crds/awx.ansible.com_awxbackups_crd.yaml | 28 ++-- .../awx.ansible.com_v1beta1_awxbackup_cr.yaml | 2 + roles/backup/README.md | 5 +- roles/backup/defaults/main.yml | 10 +- roles/backup/handlers/main.yml | 17 ++ roles/backup/store_secrets.yml | 7 - roles/backup/tasks/awx-cro.yml | 38 +++++ roles/backup/tasks/init.yml | 11 +- roles/backup/tasks/main.yml | 35 ++-- roles/backup/tasks/postgres.yml | 152 +++++++++--------- roles/backup/tasks/secrets.yml | 28 ++-- roles/backup/tasks/update_status.yml | 16 ++ roles/backup/templates/awx_object.yml.j2 | 9 ++ .../tasks/database_configuration.yml | 13 ++ roles/installer/tasks/update_status.yml | 10 -- watches.yaml | 2 +- 16 files changed, 243 insertions(+), 140 deletions(-) create mode 100644 roles/backup/handlers/main.yml delete mode 100644 roles/backup/store_secrets.yml create mode 100644 roles/backup/tasks/awx-cro.yml create mode 100644 roles/backup/tasks/update_status.yml create mode 100644 roles/backup/templates/awx_object.yml.j2 diff --git a/deploy/crds/awx.ansible.com_awxbackups_crd.yaml b/deploy/crds/awx.ansible.com_awxbackups_crd.yaml index f5303c49..5e9f9deb 100644 --- a/deploy/crds/awx.ansible.com_awxbackups_crd.yaml +++ b/deploy/crds/awx.ansible.com_awxbackups_crd.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -11,12 +12,21 @@ spec: singular: awxbackup scope: Namespaced versions: - - name: v1beta1 - schema: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true - served: true - storage: true - subresources: - status: {} + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + # TODO: Figure out how to require the tower_name field + properties: + spec: + type: object + properties: + tower_name: + description: Name of the deployment to be backed up + type: string diff --git a/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml b/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml index 7bd14a02..58ddf9eb 100644 --- a/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml +++ b/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml @@ -1,9 +1,11 @@ +--- apiVersion: awx.ansible.com/v1beta1 kind: AWXBackup metadata: name: example-awxbackup namespace: example-awx spec: + tower_name: '' tower_backup_pvc: '' tower_backup_size: '' tower_backup_storage_class: '' diff --git a/roles/backup/README.md b/roles/backup/README.md index 224a861d..8fa467d4 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -28,9 +28,12 @@ kind: AWXBackup metadata: name: awxbackup namespace: my-namespace +spec: + tower_name: mytower ``` -> The metadata.name you provide, is the name of the AWX deployment you intend to backup from (in case you have multiple in the same namespace). +Note that the `tower_name` above is the name of the AWX deployment you intend to backup from. + Finally, use `kubectl` to create the backup object in your cluster: diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 0c33617d..59310cdd 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,4 +1,6 @@ --- +# Required: specify name of tower deployment to backup from +tower_name: '' # Specify a pre-created PVC (name) to backup to tower_backup_pvc: '' @@ -10,7 +12,7 @@ tower_backup_size: '' tower_backup_storage_class: '' # Secret Names -tower_secret_key_secret: "{{ meta.name }}-secret-key" -tower_admin_password_secret: "{{ meta.name }}-admin-password" -tower_broadcast_websocket_secret: "{{ meta.name }}-broadcast-websocket" -tower_postgres_configuration_secret: "{{ meta.name }}-postgres-configuration" +tower_secret_key_secret: "{{ tower_name }}-secret-key" +tower_admin_password_secret: "{{ tower_name }}-admin-password" +tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket" +tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration" diff --git a/roles/backup/handlers/main.yml b/roles/backup/handlers/main.yml new file mode 100644 index 00000000..e6ffb379 --- /dev/null +++ b/roles/backup/handlers/main.yml @@ -0,0 +1,17 @@ +--- + +- name: Update awxbackup status + block: + - name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + + - name: Update error status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + error: "{{ error_msg }}" diff --git a/roles/backup/store_secrets.yml b/roles/backup/store_secrets.yml deleted file mode 100644 index bd900004..00000000 --- a/roles/backup/store_secrets.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -- name: Test Getting and Storing Secrets (OCP) - hosts: localhost - - tasks: - - import_tasks: tasks/secrets.yml diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml new file mode 100644 index 00000000..a2b6df47 --- /dev/null +++ b/roles/backup/tasks/awx-cro.yml @@ -0,0 +1,38 @@ +--- + +- name: Get AWX custom resource object + k8s_info: + version: v1beta1 + kind: AWX + namespace: '{{ meta.namespace }}' + name: '{{ tower_name }}' + register: _awx_cro + +- name: Set AWX object + set_fact: + _awx: "{{ _awx_cro['resources'][0] }}" + +- name: Set apiVersion + set_fact: + awx_api_version: "{{ _awx['apiVersion'] }}" + +- name: Set user specified spec + set_fact: + awx_spec: "{{ _awx['spec'] }}" + +- name: Template AWX object definition + template: + src: awx_object.yml.j2 + dest: "_secrets/awx_object.yml" + mode: '0600' + +- name: Set AWX object template file as var + set_fact: + awx_object_template: "{{ lookup('file', '_secrets/awx_object.yml') }}" + +- name: Write awx object to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ awx_object_template }}' > {{ _backup_dir }}/awx_object.yml" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 87d4cf46..85c056fd 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -19,6 +19,11 @@ when: - tower_backup_pvc != '' or tower_backup_pvc is defined +- name: Update status + set_fact: + error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + notify: "Update awxbackup status" + - name: Fail early if pvc is defined but does not exist fail: msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." @@ -29,12 +34,10 @@ set_fact: _default_backup_pvc: "{{ meta.name }}-backup-claim" +# by default, it will re-use the old pvc if already created (unless pvc is provided) - name: Set PVC to use for backup set_fact: - backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true)}}" - -# TODO: re-use the old pvc if already created (unless pvc is provided) -# TODO: allow users to configure their own storage class for dynamically creating a pvc? + backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" - name: Create PVC for backup community.kubernetes.k8s: diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 458cf9b7..dfe9c05e 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,26 +1,37 @@ --- +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + - name: Look up details for this deployment k8s_info: - api_version: 'v1beta1' # TODO: How to parameterize this? - kind: "AWX" # TODO: How to parameterize this? + api_version: "{{ api_version }}" + kind: "{{ kind }}" name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" - register: this_awx + register: this_backup - block: - - include_tasks: init.yml + - include_tasks: init.yml - - include_tasks: postgres.yml + - include_tasks: postgres.yml - - include_tasks: secrets.yml + - include_tasks: secrets.yml - # TODO: Add task to change the status on the backup CR when this runs successfully - - name: Set flag signifying this backup was successful - set_fact: - tower_backup_complete: "{{ _backup_dir }}" + - include_tasks: awx-cro.yml - - include_tasks: cleanup.yml + - name: Set flag signifying this backup was successful + set_fact: + tower_backup_complete: "{{ _backup_dir }}" + + - include_tasks: cleanup.yml when: - - this_awx['resources'][0]['status']['towerMigratedFromSecret'] is not defined + - this_backup['resources'][0]['status']['towerBackupComplete'] is not defined + +- name: Update status variables + include_tasks: update_status.yml + +# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object. See ticket diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index f17987b8..2925be17 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -1,91 +1,89 @@ --- - - name: Check for specified PostgreSQL configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ tower_postgres_configuration_secret }}' - register: _custom_pg_config_resources - when: tower_postgres_configuration_secret | length - - name: Check for default PostgreSQL configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ meta.name }}-postgres-configuration' - register: _default_pg_config_resources +- name: Check for specified PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_postgres_configuration_secret }}' + register: _custom_pg_config_resources + when: tower_postgres_configuration_secret | length - - name: Set PostgreSQL configuration - set_fact: - pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' +- name: Check for default PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_name }}-postgres-configuration' + register: _default_pg_config_resources - - name: Store Database Configuration - set_fact: - awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" - awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" - awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" - awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" - awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" +- name: Set PostgreSQL configuration + set_fact: + pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' - - name: Get the postgres pod information - k8s_info: - kind: Pod - namespace: '{{ meta.namespace }}' - label_selectors: - - "app={{ meta.name }}-{{ deployment_type }}-postgres" - register: postgres_pod - until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" - delay: 5 - retries: 60 +- name: Store Database Configuration + set_fact: + awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" - - name: Set the resource pod name as a variable. - set_fact: - postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "app={{ tower_name }}-{{ deployment_type }}-postgres" + register: postgres_pod + until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 - - name: Determine the timestamp for the backup once for all nodes - set_fact: - now: '{{ lookup("pipe", "date +%F-%T") }}' +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" - - name: Set backup directory name - set_fact: - _backup_dir: "/backups/tower-openshift-backup-{{ now }}" +- name: Determine the timestamp for the backup once for all nodes + set_fact: + now: '{{ lookup("pipe", "date +%F-%T") }}' - - name: Create directory for backup - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - mkdir -p {{ _backup_dir }} +- name: Set backup directory name + set_fact: + _backup_dir: "/backups/tower-openshift-backup-{{ now }}" - - name: Precreate file for database dump - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - touch {{ _backup_dir }}/tower.db +- name: Create directory for backup + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + mkdir -p {{ _backup_dir }} - - name: Set permissions on file for database dump - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - chmod 0600 {{ _backup_dir }}/tower.db +- name: Precreate file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + touch {{ _backup_dir }}/tower.db - - name: Set pg_dump command - set_fact: - pgdump: >- - pg_dump --clean --create - -h {{ awx_postgres_host }} - -U {{ awx_postgres_user }} - -d {{ awx_postgres_database }} - -p {{ awx_postgres_port }} +- name: Set permissions on file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + chmod 0600 {{ _backup_dir }}/tower.db - - name: Write pg_dump to backup on PVC - community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" - register: data_migration +- name: Set pg_dump command + set_fact: + pgdump: >- + pg_dump --clean --create + -h {{ awx_postgres_host }} + -U {{ awx_postgres_user }} + -d {{ awx_postgres_database }} + -p {{ awx_postgres_port }} - # TODO: Backup secret key and other secrets - look at trad tower backup pattern - # TODO: Compare final backup tar with one from a trad tower +- name: Write pg_dump to backup on PVC + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" + register: data_migration diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index ee30e690..cb062190 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -1,15 +1,10 @@ --- -# TODO: Get Secret_key value/s - -# TODO: Store Secret_key value/s in a way that can be made into another secret upon restore - -# The general idea here is that the user provides the name for the current deployment, we grab secrets based on that, then when it is restored, we restore to whatever name/namespace is specified at the time of restore - - name: Make _secrets directory file: path: "_secrets" state: directory + mode: '0700' - name: Get secret_key k8s_info: @@ -28,8 +23,9 @@ dest: "_secrets/secret_key_secret.yml" mode: '0600' -- set_fact: - secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml')}}" +- name: Set secret key template + set_fact: + secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: @@ -44,7 +40,6 @@ namespace: '{{ meta.namespace }}' name: '{{ tower_admin_password_secret }}' register: _admin_password -# TODO: check if admin_password secret name is provided, and check for that? use defaults.yml - name: Set admin_password set_fact: @@ -56,8 +51,9 @@ dest: "_secrets/admin_password_secret.yml" mode: '0600' -- set_fact: - admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml')}}" +- name: Set admin_password template + set_fact: + admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: @@ -83,8 +79,9 @@ dest: "_secrets/broadcast_websocket_secret.yml" mode: '0600' -- set_fact: - broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml')}}" +- name: Set broadcast_websocket template + set_fact: + broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: @@ -115,8 +112,9 @@ dest: "_secrets/postgres_secret.yml" mode: '0600' -- set_fact: - postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml')}}" +- name: Set postgres configuration + set_fact: + postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml') }}" - name: Write secret_key to pvc community.kubernetes.k8s_exec: diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml new file mode 100644 index 00000000..1c6ef1c6 --- /dev/null +++ b/roles/backup/tasks/update_status.yml @@ -0,0 +1,16 @@ +--- +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +# The backup directory in this status can be referenced when restoring +- name: Update Tower Backup status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerBackupComplete: "{{ _backup_dir }}" + when: tower_backup_complete is defined diff --git a/roles/backup/templates/awx_object.yml.j2 b/roles/backup/templates/awx_object.yml.j2 new file mode 100644 index 00000000..b0a539d9 --- /dev/null +++ b/roles/backup/templates/awx_object.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: '{{ awx_api_version }}' +kind: AWX +metadata: +{% raw %} + name: '{{ meta.name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +spec: {{ awx_spec }} diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index ec476f7c..18d408fd 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -110,6 +110,19 @@ awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}" +# - name: Set apiVersion and kind variables +# set_fact: +# api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' +# kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' +# +# - name: Look up details for this deployment +# k8s_info: +# api_version: "{{ api_version }}" +# kind: "{{ kind }}" +# name: "{{ meta.name }}" +# namespace: "{{ meta.namespace }}" +# register: this_awx + - name: Look up details for this deployment k8s_info: api_version: 'v1beta1' # TODO: How to parameterize this? diff --git a/roles/installer/tasks/update_status.yml b/roles/installer/tasks/update_status.yml index 93068f1c..ec4b3d54 100644 --- a/roles/installer/tasks/update_status.yml +++ b/roles/installer/tasks/update_status.yml @@ -73,13 +73,3 @@ status: towerMigratedFromSecret: "{{ tower_migrated_from_secret }}" when: tower_migrated_from_secret is defined - -- name: Update Tower Backup status - operator_sdk.util.k8s_status: - api_version: '{{ api_version }}' - kind: "{{ kind }}" - name: "{{ meta.name }}" - namespace: "{{ meta.namespace }}" - status: - towerBackupComplete: "{{ _backup_dir }}" - when: tower_backup_complete is defined diff --git a/watches.yaml b/watches.yaml index 01114c09..3d59e606 100644 --- a/watches.yaml +++ b/watches.yaml @@ -9,6 +9,6 @@ - version: v1beta1 group: awx.ansible.com - kind: Backup + kind: AWXBackup role: backup reconcilePeriod: 360m From 250ff960bd31cc33a09c9a85c4958f8a9522782e Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 31 Mar 2021 01:04:51 -0400 Subject: [PATCH 14/43] Add awxbackup CRD creation to molecule to get tests passing --- molecule/default/prepare.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index a5edb8fb..bfa93fbb 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -11,10 +11,14 @@ - "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/ansible/group_vars/all" tasks: - - name: Create Custom Resource Definition + - name: Create AWX Custom Resource Definition k8s: definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_crd.yaml'])) }}" + - name: Create AWXBackup Custom Resource Definition + k8s: + definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx.ansible.com_awxbackups_crd.yaml'])) }}" + - name: Ensure specified namespace is present k8s: api_version: v1 From 6bc149bae236243789c6f82d56f6b448721d53a6 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 31 Mar 2021 10:33:30 -0400 Subject: [PATCH 15/43] template awxbackup crd into awx-operator.yml for easy deployment --- ansible/chain-operator-files.yml | 8 +- ansible/templates/awx-operator.yaml.j2 | 4 + .../templates/awxbackup_crd.yml.j2 | 0 deploy/awx-operator.yaml | 177 +++--------------- ...ckup_cr.yaml => awxbackup_v1beta1_cr.yaml} | 0 deploy/crds/awxbackup_v1beta1_crd.yaml | 32 ++++ molecule/default/prepare.yml | 2 +- molecule/test-local/converge.yml | 3 +- .../tasks/database_configuration.yml | 20 +- 9 files changed, 81 insertions(+), 165 deletions(-) rename deploy/crds/awx.ansible.com_awxbackups_crd.yaml => ansible/templates/awxbackup_crd.yml.j2 (100%) rename deploy/crds/{awx.ansible.com_v1beta1_awxbackup_cr.yaml => awxbackup_v1beta1_cr.yaml} (100%) create mode 100644 deploy/crds/awxbackup_v1beta1_crd.yaml diff --git a/ansible/chain-operator-files.yml b/ansible/chain-operator-files.yml index 03f460bf..e412f28a 100644 --- a/ansible/chain-operator-files.yml +++ b/ansible/chain-operator-files.yml @@ -6,12 +6,18 @@ gather_facts: false tasks: - - name: Template CRD + - name: Template AWX CRD template: src: crd.yml.j2 dest: "{{ playbook_dir }}/../deploy/crds/awx_v1beta1_crd.yaml" mode: '0644' + - name: Template AWXBackup CRD + template: + src: awxbackup_crd.yml.j2 + dest: "{{ playbook_dir }}/../deploy/crds/awxbackup_v1beta1_crd.yaml" + mode: '0644' + - name: Template awx-operator.yaml template: src: awx-operator.yaml.j2 diff --git a/ansible/templates/awx-operator.yaml.j2 b/ansible/templates/awx-operator.yaml.j2 index 6d1364ba..e7d164ee 100644 --- a/ansible/templates/awx-operator.yaml.j2 +++ b/ansible/templates/awx-operator.yaml.j2 @@ -10,3 +10,7 @@ {% include 'service_account.yml.j2' %} {% include 'operator.yml.j2' %} + +{% include 'crd.yml.j2' %} + +{% include 'awxbackup_crd.yml.j2' %} diff --git a/deploy/crds/awx.ansible.com_awxbackups_crd.yaml b/ansible/templates/awxbackup_crd.yml.j2 similarity index 100% rename from deploy/crds/awx.ansible.com_awxbackups_crd.yaml rename to ansible/templates/awxbackup_crd.yml.j2 diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index f87b157d..dc4252da 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -80,7 +80,7 @@ rules: - awx.ansible.com resources: - '*' - - backups + - awxbackups verbs: - '*' @@ -527,151 +527,34 @@ spec: type: object --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: - creationTimestamp: null - name: awx-operator -rules: - - apiGroups: - - route.openshift.io - resources: - - routes - - routes/custom-host - verbs: - - '*' - - apiGroups: - - "" - - "rbac.authorization.k8s.io" - resources: - - pods - - services - - services/finalizers - - serviceaccounts - - endpoints - - persistentvolumeclaims - - events - - configmaps - - secrets - - roles - - rolebindings - verbs: - - '*' - - apiGroups: - - apps - - extensions - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - - ingresses - verbs: - - '*' - - apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create - - apiGroups: - - apps - resourceNames: - - awx-operator - resources: - - deployments/finalizers - verbs: - - update - - apiGroups: - - apps - resources: - - deployments/scale - - statefulsets/scale - verbs: - - patch - - apiGroups: - - "" - resources: - - pods/exec - verbs: - - create - - get - - apiGroups: - - apps - resources: - - replicasets - verbs: - - get - - apiGroups: - - awx.ansible.com - resources: - - '*' - verbs: - - '*' - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: awx-operator -subjects: - - kind: ServiceAccount - name: awx-operator - namespace: default -roleRef: - kind: ClusterRole - name: awx-operator - apiGroup: rbac.authorization.k8s.io - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: awx-operator - namespace: default - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: awx-operator + name: awxbackups.awx.ansible.com spec: - replicas: 1 - selector: - matchLabels: - name: awx-operator - template: - metadata: - labels: - name: awx-operator - spec: - serviceAccountName: awx-operator - containers: - - name: awx-operator - image: "quay.io/ansible/awx-operator:0.8.0" - imagePullPolicy: "Always" - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - env: - # Watch all namespaces (cluster-scoped). - - name: WATCH_NAMESPACE - value: "" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: awx-operator - - name: ANSIBLE_GATHERING - value: explicit - livenessProbe: - httpGet: - path: /healthz - port: 6789 - initialDelaySeconds: 15 - periodSeconds: 20 - volumes: - - name: runner - emptyDir: {} + group: awx.ansible.com + names: + kind: AWXBackup + listKind: AWXBackupList + plural: awxbackups + singular: awxbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + # TODO: Figure out how to require the tower_name field + properties: + spec: + type: object + properties: + tower_name: + description: Name of the deployment to be backed up + type: string diff --git a/deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml b/deploy/crds/awxbackup_v1beta1_cr.yaml similarity index 100% rename from deploy/crds/awx.ansible.com_v1beta1_awxbackup_cr.yaml rename to deploy/crds/awxbackup_v1beta1_cr.yaml diff --git a/deploy/crds/awxbackup_v1beta1_crd.yaml b/deploy/crds/awxbackup_v1beta1_crd.yaml new file mode 100644 index 00000000..5e9f9deb --- /dev/null +++ b/deploy/crds/awxbackup_v1beta1_crd.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxbackups.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXBackup + listKind: AWXBackupList + plural: awxbackups + singular: awxbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + # TODO: Figure out how to require the tower_name field + properties: + spec: + type: object + properties: + tower_name: + description: Name of the deployment to be backed up + type: string diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index bfa93fbb..a738ac1f 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -17,7 +17,7 @@ - name: Create AWXBackup Custom Resource Definition k8s: - definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx.ansible.com_awxbackups_crd.yaml'])) }}" + definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxbackup_v1beta1_crd.yaml'])) }}" - name: Ensure specified namespace is present k8s: diff --git a/molecule/test-local/converge.yml b/molecule/test-local/converge.yml index 58790fe0..ee5f1a35 100644 --- a/molecule/test-local/converge.yml +++ b/molecule/test-local/converge.yml @@ -29,10 +29,9 @@ operator_image: awx.ansible.com/awx-operator operator_version: testing custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_molecule.yaml'])) | from_yaml }}" + tasks: - - block: - - name: Delete the Operator Deployment k8s: state: absent diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index 18d408fd..1f52509d 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -110,23 +110,15 @@ awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}" -# - name: Set apiVersion and kind variables -# set_fact: -# api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' -# kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' -# -# - name: Look up details for this deployment -# k8s_info: -# api_version: "{{ api_version }}" -# kind: "{{ kind }}" -# name: "{{ meta.name }}" -# namespace: "{{ meta.namespace }}" -# register: this_awx +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - name: Look up details for this deployment k8s_info: - api_version: 'v1beta1' # TODO: How to parameterize this? - kind: "AWX" # TODO: How to parameterize this? + api_version: "{{ api_version }}" + kind: "{{ kind }}" name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" register: this_awx From 80c8d87f71c4eac4531a0892f7988e9af192452f Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 31 Mar 2021 15:21:42 -0400 Subject: [PATCH 16/43] Create an event when pvc is not set to alert the user --- ansible/templates/awxbackup_crd.yml.j2 | 24 +++++++++++++++++++++++- deploy/awx-operator.yaml | 24 +++++++++++++++++++++++- deploy/crds/awxbackup_v1beta1_crd.yaml | 24 +++++++++++++++++++++++- roles/backup/handlers/main.yml | 17 ----------------- roles/backup/tasks/error_handling.yml | 16 ++++++++++++++++ roles/backup/tasks/init.yml | 24 +++++++++++++++--------- roles/backup/templates/event.yml.j2 | 17 +++++++++++++++++ watches.yaml | 1 - 8 files changed, 117 insertions(+), 30 deletions(-) delete mode 100644 roles/backup/handlers/main.yml create mode 100644 roles/backup/tasks/error_handling.yml create mode 100644 roles/backup/templates/event.yml.j2 diff --git a/ansible/templates/awxbackup_crd.yml.j2 b/ansible/templates/awxbackup_crd.yml.j2 index 5e9f9deb..d0ad49a9 100644 --- a/ansible/templates/awxbackup_crd.yml.j2 +++ b/ansible/templates/awxbackup_crd.yml.j2 @@ -22,7 +22,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true description: Schema validation for the AWXBackup CRD - # TODO: Figure out how to require the tower_name field properties: spec: type: object @@ -30,3 +29,26 @@ spec: tower_name: description: Name of the deployment to be backed up type: string + tower_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + tower_backup_size: + description: Size of PVC + type: string + tower_backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + oneOf: + - required: ["tower_name"] diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index dc4252da..8b80f713 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -550,7 +550,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true description: Schema validation for the AWXBackup CRD - # TODO: Figure out how to require the tower_name field properties: spec: type: object @@ -558,3 +557,26 @@ spec: tower_name: description: Name of the deployment to be backed up type: string + tower_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + tower_backup_size: + description: Size of PVC + type: string + tower_backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + oneOf: + - required: ["tower_name"] diff --git a/deploy/crds/awxbackup_v1beta1_crd.yaml b/deploy/crds/awxbackup_v1beta1_crd.yaml index 5e9f9deb..d0ad49a9 100644 --- a/deploy/crds/awxbackup_v1beta1_crd.yaml +++ b/deploy/crds/awxbackup_v1beta1_crd.yaml @@ -22,7 +22,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true description: Schema validation for the AWXBackup CRD - # TODO: Figure out how to require the tower_name field properties: spec: type: object @@ -30,3 +29,26 @@ spec: tower_name: description: Name of the deployment to be backed up type: string + tower_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + tower_backup_size: + description: Size of PVC + type: string + tower_backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + oneOf: + - required: ["tower_name"] diff --git a/roles/backup/handlers/main.yml b/roles/backup/handlers/main.yml deleted file mode 100644 index e6ffb379..00000000 --- a/roles/backup/handlers/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- - -- name: Update awxbackup status - block: - - name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - - - name: Update error status - operator_sdk.util.k8s_status: - api_version: '{{ api_version }}' - kind: "{{ kind }}" - name: "{{ meta.name }}" - namespace: "{{ meta.namespace }}" - status: - error: "{{ error_msg }}" diff --git a/roles/backup/tasks/error_handling.yml b/roles/backup/tasks/error_handling.yml new file mode 100644 index 00000000..f3361d6c --- /dev/null +++ b/roles/backup/tasks/error_handling.yml @@ -0,0 +1,16 @@ +--- + +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +- name: Determine the timestamp + set_fact: + now: '{{ lookup("pipe", "date +%FT%TZ") }}' + +- name: Emit ocp event with error + community.kubernetes.k8s: + kind: Event + namespace: "{{ meta.namespace }}" + template: "event.yml.j2" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 85c056fd..68fb305c 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -17,17 +17,23 @@ namespace: "{{ meta.namespace }}" register: provided_pvc when: - - tower_backup_pvc != '' or tower_backup_pvc is defined + - tower_backup_pvc != '' -- name: Update status - set_fact: - error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." - notify: "Update awxbackup status" +- name: Surface error to user + block: + - name: Set error message + set_fact: + error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." -- name: Fail early if pvc is defined but does not exist - fail: - msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." - when: provided_pvc.resources | length == 0 + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + when: + - tower_backup_pvc != '' + - provided_pvc.resources | length == 0 # If tower_backup_pvc is defined, use in management-pod.yml.j2 - name: Set default pvc name diff --git a/roles/backup/templates/event.yml.j2 b/roles/backup/templates/event.yml.j2 new file mode 100644 index 00000000..ead6aea4 --- /dev/null +++ b/roles/backup/templates/event.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Event +metadata: + name: backup-error.{{ now }} + namespace: {{ meta.namespace }} +involvedObject: + apiVersion: awx.ansible.com/v1beta1 + kind: {{ kind }} + name: {{ meta.name }} + namespace: {{ meta.namespace }} +message: {{ error_msg }} +reason: BackupFailed +type: Warning +firstTimestamp: {{ now }} +lastTimestamp: {{ now }} +count: 1 diff --git a/watches.yaml b/watches.yaml index 3d59e606..1b294dc3 100644 --- a/watches.yaml +++ b/watches.yaml @@ -11,4 +11,3 @@ group: awx.ansible.com kind: AWXBackup role: backup - reconcilePeriod: 360m From 8467209d3503845eb6d6952cfd35f8d97ba98ab3 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 1 Apr 2021 14:52:22 -0400 Subject: [PATCH 17/43] init restore --- ansible/chain-operator-files.yml | 6 ++ ansible/templates/awx-operator.yaml.j2 | 2 + ansible/templates/awxrestore_crd.yml.j2 | 51 +++++++++++ deploy/awx-operator.yaml | 55 ++++++++++++ deploy/crds/awxbackup_v1beta1_cr.yaml | 12 --- deploy/crds/awxrestore_v1beta1_crd.yaml | 51 +++++++++++ molecule/default/prepare.yml | 4 + roles/backup/tasks/secrets.yml | 4 +- .../templates/admin_password_secret.yml.j2 | 2 +- roles/backup/templates/awx_object.yml.j2 | 2 +- .../broadcast_websocket_secret.yml.j2 | 2 +- roles/backup/templates/event.yml.j2 | 4 +- roles/backup/templates/postgres_secret.yml.j2 | 2 +- .../backup/templates/secret_key_secret.yml.j2 | 2 +- roles/restore/.secrets.yml.swp | Bin 0 -> 12288 bytes roles/restore/README.md | 82 +++++++++++++++++ roles/restore/defaults/main.yml | 17 ++++ roles/restore/meta/main.yml | 30 +++++++ roles/restore/tasks/cleanup.yml | 9 ++ roles/restore/tasks/error_handling.yml | 16 ++++ roles/restore/tasks/init.yml | 55 ++++++++++++ roles/restore/tasks/init_awx.yml | 12 +++ roles/restore/tasks/main.yml | 39 ++++++++ roles/restore/tasks/postgres.yml | 85 ++++++++++++++++++ roles/restore/tasks/preflight.yml | 27 ++++++ roles/restore/tasks/secrets.yml | 21 +++++ roles/restore/tasks/update_status.yml | 15 ++++ roles/restore/templates/event.yml.j2 | 17 ++++ roles/restore/templates/management-pod.yml.j2 | 22 +++++ roles/restore/vars/main.yml | 4 + watches.yaml | 5 ++ 31 files changed, 634 insertions(+), 21 deletions(-) create mode 100644 ansible/templates/awxrestore_crd.yml.j2 delete mode 100644 deploy/crds/awxbackup_v1beta1_cr.yaml create mode 100644 deploy/crds/awxrestore_v1beta1_crd.yaml create mode 100644 roles/restore/.secrets.yml.swp create mode 100644 roles/restore/README.md create mode 100644 roles/restore/defaults/main.yml create mode 100644 roles/restore/meta/main.yml create mode 100644 roles/restore/tasks/cleanup.yml create mode 100644 roles/restore/tasks/error_handling.yml create mode 100644 roles/restore/tasks/init.yml create mode 100644 roles/restore/tasks/init_awx.yml create mode 100644 roles/restore/tasks/main.yml create mode 100644 roles/restore/tasks/postgres.yml create mode 100644 roles/restore/tasks/preflight.yml create mode 100644 roles/restore/tasks/secrets.yml create mode 100644 roles/restore/tasks/update_status.yml create mode 100644 roles/restore/templates/event.yml.j2 create mode 100644 roles/restore/templates/management-pod.yml.j2 create mode 100644 roles/restore/vars/main.yml diff --git a/ansible/chain-operator-files.yml b/ansible/chain-operator-files.yml index e412f28a..23cfc05a 100644 --- a/ansible/chain-operator-files.yml +++ b/ansible/chain-operator-files.yml @@ -18,6 +18,12 @@ dest: "{{ playbook_dir }}/../deploy/crds/awxbackup_v1beta1_crd.yaml" mode: '0644' + - name: Template AWXRestore CRD + template: + src: awxrestore_crd.yml.j2 + dest: "{{ playbook_dir }}/../deploy/crds/awxrestore_v1beta1_crd.yaml" + mode: '0644' + - name: Template awx-operator.yaml template: src: awx-operator.yaml.j2 diff --git a/ansible/templates/awx-operator.yaml.j2 b/ansible/templates/awx-operator.yaml.j2 index e7d164ee..31fe55b6 100644 --- a/ansible/templates/awx-operator.yaml.j2 +++ b/ansible/templates/awx-operator.yaml.j2 @@ -14,3 +14,5 @@ {% include 'crd.yml.j2' %} {% include 'awxbackup_crd.yml.j2' %} + +{% include 'awxbackup_crd.yml.j2' %} diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 new file mode 100644 index 00000000..39b669a6 --- /dev/null +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -0,0 +1,51 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxrestores.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXRestore + listKind: AWXRestoreList + plural: awxrestores + singular: awxrestore + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + properties: + spec: + type: object + properties: + tower_name: + description: Name of the deployment to be restored to + type: string + tower_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + tower_backup_dir: + description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + oneOf: + - required: ["tower_name", "tower_backup_pvc"] diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 8b80f713..6fc80e38 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -580,3 +580,58 @@ spec: type: string oneOf: - required: ["tower_name"] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxbackups.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXBackup + listKind: AWXBackupList + plural: awxbackups + singular: awxbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + properties: + spec: + type: object + properties: + tower_name: + description: Name of the deployment to be backed up + type: string + tower_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + tower_backup_size: + description: Size of PVC + type: string + tower_backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + oneOf: + - required: ["tower_name"] diff --git a/deploy/crds/awxbackup_v1beta1_cr.yaml b/deploy/crds/awxbackup_v1beta1_cr.yaml deleted file mode 100644 index 58ddf9eb..00000000 --- a/deploy/crds/awxbackup_v1beta1_cr.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: awx.ansible.com/v1beta1 -kind: AWXBackup -metadata: - name: example-awxbackup - namespace: example-awx -spec: - tower_name: '' - tower_backup_pvc: '' - tower_backup_size: '' - tower_backup_storage_class: '' - tower_postgres_configuration_secret: '' diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml new file mode 100644 index 00000000..39b669a6 --- /dev/null +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxrestores.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXRestore + listKind: AWXRestoreList + plural: awxrestores + singular: awxrestore + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + properties: + spec: + type: object + properties: + tower_name: + description: Name of the deployment to be restored to + type: string + tower_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + tower_backup_dir: + description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + oneOf: + - required: ["tower_name", "tower_backup_pvc"] diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index a738ac1f..76ce4ad0 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -19,6 +19,10 @@ k8s: definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxbackup_v1beta1_crd.yaml'])) }}" + - name: Create AWXBackup Custom Resource Definition + k8s: + definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxrestore_v1beta1_crd.yaml'])) }}" + - name: Ensure specified namespace is present k8s: api_version: v1 diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index cb062190..d5f0105e 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -83,7 +83,7 @@ set_fact: broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml') }}" -- name: Write secret_key to pvc +- name: Write broadcast_websocket definition to pvc community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" @@ -116,7 +116,7 @@ set_fact: postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml') }}" -- name: Write secret_key to pvc +- name: Write postgres configuration to pvc community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" diff --git a/roles/backup/templates/admin_password_secret.yml.j2 b/roles/backup/templates/admin_password_secret.yml.j2 index 3d1df46e..8adc8177 100644 --- a/roles/backup/templates/admin_password_secret.yml.j2 +++ b/roles/backup/templates/admin_password_secret.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: {% raw %} - name: '{{ meta.name }}' + name: '{{ tower_name }}' namespace: '{{ meta.namespace }}' {% endraw %} stringData: diff --git a/roles/backup/templates/awx_object.yml.j2 b/roles/backup/templates/awx_object.yml.j2 index b0a539d9..fd4e9d92 100644 --- a/roles/backup/templates/awx_object.yml.j2 +++ b/roles/backup/templates/awx_object.yml.j2 @@ -3,7 +3,7 @@ apiVersion: '{{ awx_api_version }}' kind: AWX metadata: {% raw %} - name: '{{ meta.name }}' + name: '{{ tower_name }}' namespace: '{{ meta.namespace }}' {% endraw %} spec: {{ awx_spec }} diff --git a/roles/backup/templates/broadcast_websocket_secret.yml.j2 b/roles/backup/templates/broadcast_websocket_secret.yml.j2 index c8d4cc2f..e590ab06 100644 --- a/roles/backup/templates/broadcast_websocket_secret.yml.j2 +++ b/roles/backup/templates/broadcast_websocket_secret.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: {% raw %} - name: '{{ meta.name }}-broadcast-websocket' + name: '{{ tower_name }}-broadcast-websocket' namespace: '{{ meta.namespace }}' {% endraw %} stringData: diff --git a/roles/backup/templates/event.yml.j2 b/roles/backup/templates/event.yml.j2 index ead6aea4..3670cba3 100644 --- a/roles/backup/templates/event.yml.j2 +++ b/roles/backup/templates/event.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Event metadata: - name: backup-error.{{ now }} + name: restore-error.{{ now }} namespace: {{ meta.namespace }} involvedObject: apiVersion: awx.ansible.com/v1beta1 @@ -10,7 +10,7 @@ involvedObject: name: {{ meta.name }} namespace: {{ meta.namespace }} message: {{ error_msg }} -reason: BackupFailed +reason: RestoreFailed type: Warning firstTimestamp: {{ now }} lastTimestamp: {{ now }} diff --git a/roles/backup/templates/postgres_secret.yml.j2 b/roles/backup/templates/postgres_secret.yml.j2 index b1ad1f6b..f5fa75d1 100644 --- a/roles/backup/templates/postgres_secret.yml.j2 +++ b/roles/backup/templates/postgres_secret.yml.j2 @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: {% raw %} - name: '{{ meta.name }}-postgres-configuration' + name: '{{ tower_name }}-postgres-configuration' namespace: '{{ meta.namespace }}' {% endraw %} stringData: diff --git a/roles/backup/templates/secret_key_secret.yml.j2 b/roles/backup/templates/secret_key_secret.yml.j2 index febc23ee..ae60b37f 100644 --- a/roles/backup/templates/secret_key_secret.yml.j2 +++ b/roles/backup/templates/secret_key_secret.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: {% raw %} - name: '{{ meta.name }}' + name: '{{ tower_name }}' namespace: '{{ meta.namespace }}' {% endraw %} stringData: diff --git a/roles/restore/.secrets.yml.swp b/roles/restore/.secrets.yml.swp new file mode 100644 index 0000000000000000000000000000000000000000..3209698938d752dc750d6955437c6f5d7b9d0be3 GIT binary patch literal 12288 zcmeI&Jxatt6bJCPR@;c8jaZFZ1cYpMBZ34Q!N$T$OeN@S_MtJG4`xQ!1wZcL0X%_M z@Cw$JHr~L(7mba@;+6~lfj`4AZ-(U8g}i<@J~&i6oseWK60NjP&Up2mo_9AXxzGch z+j?i>XzKh?X|hR$_f27-M>l?1vC*zHfhmV$;!5eBZE@2730IZP{_ kubectl apply -f restore-awx.yml +``` + +This will create a new deployment and restore your backup to it. + + +Role Variables +-------------- + +A custom, pre-created pvc can be used by setting the following variables. + +``` +tower_backup_pvc: 'awx-backup-volume-claim' +``` + + +If a custom postgres configuration secret was used when deploying AWX, it must be set: + +``` +tower_postgres_configuration_secret: 'awx-postgres-configuration' +``` + + +Testing +---------------- + +You can test this role directly by creating and running the following playbook with the appropriate variables: + +``` +--- +- name: Backup Tower + hosts: localhost + gather_facts: false + roles: + - backup +``` + +License +------- + +MIT diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml new file mode 100644 index 00000000..67a38b79 --- /dev/null +++ b/roles/restore/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# Required: specify name of tower deployment to restore to +tower_name: '' + +# Required: specify a pre-created PVC (name) to restore from +tower_backup_pvc: '' + +# Required: backup name, found on the awxbackup object +tower_backup_dir: '' + +# TODO: Should we add a unique id at the end of the secret when backing up, then use it here? +# or will that make future backups more complicated because the user will have to specify the names of all the secrets? +# Names of any secrets you want to use instead of the ones in the backup +tower_secret_key_secret: "{{ tower_name }}-secret-key" +tower_admin_password_secret: "{{ tower_name }}-admin-password" +tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket" +tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration" diff --git a/roles/restore/meta/main.yml b/roles/restore/meta/main.yml new file mode 100644 index 00000000..f9e95b3b --- /dev/null +++ b/roles/restore/meta/main.yml @@ -0,0 +1,30 @@ +--- +galaxy_info: + author: Ansible + description: AWX role for AWX Operator for Kubernetes. + company: Red Hat, Inc. + + license: MIT + + min_ansible_version: 2.8 + + platforms: + - name: EL + versions: + - all + - name: Debian + versions: + - all + + galaxy_tags: + - tower + - awx + - ansible + - restore + - automation + +dependencies: [] + +collections: + - community.kubernetes + - operator_sdk.util diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml new file mode 100644 index 00000000..9976a8c9 --- /dev/null +++ b/roles/restore/tasks/cleanup.yml @@ -0,0 +1,9 @@ +--- + +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ meta.namespace }}" + state: absent + force: true diff --git a/roles/restore/tasks/error_handling.yml b/roles/restore/tasks/error_handling.yml new file mode 100644 index 00000000..f3361d6c --- /dev/null +++ b/roles/restore/tasks/error_handling.yml @@ -0,0 +1,16 @@ +--- + +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +- name: Determine the timestamp + set_fact: + now: '{{ lookup("pipe", "date +%FT%TZ") }}' + +- name: Emit ocp event with error + community.kubernetes.k8s: + kind: Event + namespace: "{{ meta.namespace }}" + template: "event.yml.j2" diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml new file mode 100644 index 00000000..009fba00 --- /dev/null +++ b/roles/restore/tasks/init.yml @@ -0,0 +1,55 @@ +--- + +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ meta.namespace }}" + state: absent + force: true + wait: true + +# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. +- name: Check provided PVC exists + k8s_info: + name: "{{ tower_backup_pvc }}" + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + register: provided_pvc + when: + - tower_backup_pvc != '' + +- name: Surface error to user + block: + - name: Set error message + set_fact: + error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + when: + - tower_backup_pvc != '' + - provided_pvc.resources | length == 0 + +# If tower_backup_pvc is defined, use in management-pod.yml.j2 +- name: Set default pvc name + set_fact: + _default_backup_pvc: "{{ meta.name }}-backup-claim" + +# by default, it will re-use the old pvc if already created (unless pvc is provided) +- name: Set PVC to use for backup + set_fact: + backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" + +- name: Create management pod from templated deployment config + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Deployment + namespace: "{{ meta.namespace }}" + state: present + template: "management-pod.yml.j2" + wait: true diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/init_awx.yml new file mode 100644 index 00000000..1e8c268c --- /dev/null +++ b/roles/restore/tasks/init_awx.yml @@ -0,0 +1,12 @@ +--- +- name: Deploy AWX + k8s: + state: "{{ state | default('present') }}" + namespace: "{{ tower_namespace | default('default') }}" + apply: yes + wait: yes + template: "{{ tower_backup_dir }}/awx_object.yml" + +# TODO: Add logic to allow users to provide override values here, +# or to specify spec values that were not in the backed up AWX object. +# This may involve changing how we back up the spec section of the AWX object diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml new file mode 100644 index 00000000..df03f3d1 --- /dev/null +++ b/roles/restore/tasks/main.yml @@ -0,0 +1,39 @@ +--- + +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +- name: Look up details for this deployment + k8s_info: + api_version: "{{ api_version }}" + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + register: this_restore + +- block: + - include_tasks: preflight.yml + + - include_tasks: init_awx.yml + + - include_tasks: init.yml + + # - include_tasks: postgres.yml + # + # - include_tasks: secrets.yml + + - name: Set flag signifying this backup was successful + set_fact: + tower_backup_complete: True + + - include_tasks: cleanup.yml + + when: + - this_restore['resources'][0]['status']['towerRestoreComplete'] is not defined + +- name: Update status variables + include_tasks: update_status.yml + +# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object. See ticket diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml new file mode 100644 index 00000000..e6d375e3 --- /dev/null +++ b/roles/restore/tasks/postgres.yml @@ -0,0 +1,85 @@ +--- + +- name: Check for specified PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_postgres_configuration_secret }}' + register: _custom_pg_config_resources + when: tower_postgres_configuration_secret | length + +- name: Check for default PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_name }}-postgres-configuration' + register: _default_pg_config_resources + +- name: Set PostgreSQL configuration + set_fact: + pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' + +- name: Store Database Configuration + set_fact: + awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "app={{ tower_name }}-{{ deployment_type }}-postgres" + register: postgres_pod + until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 + +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" + + +# TODO: Add postgres restore logic, just pipe the pg_dump from PVC to db via psql + +# +# - name: Create directory for backup +# community.kubernetes.k8s_exec: +# namespace: "{{ meta.namespace }}" +# pod: "{{ meta.name }}-db-management" +# command: >- +# mkdir -p {{ _backup_dir }} +# +# - name: Precreate file for database dump +# community.kubernetes.k8s_exec: +# namespace: "{{ meta.namespace }}" +# pod: "{{ meta.name }}-db-management" +# command: >- +# touch {{ _backup_dir }}/tower.db +# +# - name: Set permissions on file for database dump +# community.kubernetes.k8s_exec: +# namespace: "{{ meta.namespace }}" +# pod: "{{ meta.name }}-db-management" +# command: >- +# chmod 0600 {{ _backup_dir }}/tower.db +# +# - name: Set pg_dump command +# set_fact: +# pgdump: >- +# pg_dump --clean --create +# -h {{ awx_postgres_host }} +# -U {{ awx_postgres_user }} +# -d {{ awx_postgres_database }} +# -p {{ awx_postgres_port }} +# +# - name: Write pg_dump to backup on PVC +# community.kubernetes.k8s_exec: +# namespace: "{{ meta.namespace }}" +# pod: "{{ meta.name }}-db-management" +# command: >- +# bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" +# register: data_migration diff --git a/roles/restore/tasks/preflight.yml b/roles/restore/tasks/preflight.yml new file mode 100644 index 00000000..108162ec --- /dev/null +++ b/roles/restore/tasks/preflight.yml @@ -0,0 +1,27 @@ +--- + +# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. +- name: Check provided PVC exists + k8s_info: + name: "{{ tower_backup_pvc }}" + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + register: provided_pvc + when: + - tower_backup_pvc != '' + +- name: Surface error to user + block: + - name: Set error message + set_fact: + error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + when: + - tower_backup_pvc != '' + - provided_pvc.resources | length == 0 diff --git a/roles/restore/tasks/secrets.yml b/roles/restore/tasks/secrets.yml new file mode 100644 index 00000000..705cf133 --- /dev/null +++ b/roles/restore/tasks/secrets.yml @@ -0,0 +1,21 @@ +--- + +- name: Create secret_key secret + k8s: + definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'secret_key_secret.yml'])) }}" + +- name: Create admin_password secret + k8s: + definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'admin_password_secret.yml'])) }}" + +- name: Create broadcast_websocket secret + k8s: + definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'broadcast_websocket_secret.yml'])) }}" + +- name: Create postgres configuration secret + k8s: + definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'postgres_secret.yml'])) }}" + +- name: Create secret_key secret + k8s: + definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'secret_key_secret.yml'])) }}" diff --git a/roles/restore/tasks/update_status.yml b/roles/restore/tasks/update_status.yml new file mode 100644 index 00000000..f056a1ac --- /dev/null +++ b/roles/restore/tasks/update_status.yml @@ -0,0 +1,15 @@ +--- +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +- name: Update Tower Backup status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerRestoreComplete: True + when: tower_restore_complete is defined diff --git a/roles/restore/templates/event.yml.j2 b/roles/restore/templates/event.yml.j2 new file mode 100644 index 00000000..ead6aea4 --- /dev/null +++ b/roles/restore/templates/event.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Event +metadata: + name: backup-error.{{ now }} + namespace: {{ meta.namespace }} +involvedObject: + apiVersion: awx.ansible.com/v1beta1 + kind: {{ kind }} + name: {{ meta.name }} + namespace: {{ meta.namespace }} +message: {{ error_msg }} +reason: BackupFailed +type: Warning +firstTimestamp: {{ now }} +lastTimestamp: {{ now }} +count: 1 diff --git a/roles/restore/templates/management-pod.yml.j2 b/roles/restore/templates/management-pod.yml.j2 new file mode 100644 index 00000000..87ddff8e --- /dev/null +++ b/roles/restore/templates/management-pod.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ meta.name }}-db-management + namespace: {{ meta.namespace }} +spec: + containers: + - name: {{ meta.name }}-db-management + image: "{{ tower_postgres_image }}" + imagePullPolicy: Always + command: ["sleep", "infinity"] + volumeMounts: + - name: {{ meta.name }}-backup + mountPath: /backups + readOnly: false + volumes: + - name: {{ meta.name }}-backup + persistentVolumeClaim: + claimName: {{ backup_pvc }} + readOnly: false + restartPolicy: Never diff --git a/roles/restore/vars/main.yml b/roles/restore/vars/main.yml new file mode 100644 index 00000000..851b98f0 --- /dev/null +++ b/roles/restore/vars/main.yml @@ -0,0 +1,4 @@ +--- + +deployment_type: "awx" +tower_postgres_image: postgres:12 diff --git a/watches.yaml b/watches.yaml index 1b294dc3..0a5f795b 100644 --- a/watches.yaml +++ b/watches.yaml @@ -11,3 +11,8 @@ group: awx.ansible.com kind: AWXBackup role: backup + +- version: v1beta1 + group: awx.ansible.com + kind: AWXRestore + role: restore From 8422f6fbd95322488420e0f4eaf01cd28582b22e Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 1 Apr 2021 22:45:35 -0400 Subject: [PATCH 18/43] rename db task vars with awx instead of tower for consistency --- roles/installer/tasks/migrate_data.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index aeea94ec..698ba081 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -2,11 +2,11 @@ - name: Store Database Configuration set_fact: - tower_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}" - tower_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}" - tower_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}" - tower_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}" - tower_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}" + awx_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}" - name: Get the postgres pod information k8s_info: @@ -31,10 +31,10 @@ set_fact: pgdump: >- pg_dump --clean --create - -h {{ tower_old_postgres_host }} - -U {{ tower_old_postgres_user }} - -d {{ tower_old_postgres_database }} - -p {{ tower_old_postgres_port }} + -h {{ awx_old_postgres_host }} + -U {{ awx_old_postgres_user }} + -d {{ awx_old_postgres_database }} + -p {{ awx_old_postgres_port }} - name: Set pg_restore command set_fact: @@ -50,7 +50,7 @@ command: | bash -c """ set -e -o pipefail - PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} + PGPASSWORD={{ awx_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} echo 'Successful' """ register: data_migration @@ -58,4 +58,4 @@ - name: Set flag signifying that this instance has been migrated set_fact: - tower_migrated_from_secret: "{{ tower_old_postgres_configuration_secret }}" + tower_migrated_from_secret: "{{ awx_old_postgres_configuration_secret }}" From 0580398c909337f867ea0105f69e868c7ff0166e Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 1 Apr 2021 22:57:37 -0400 Subject: [PATCH 19/43] Finish db restore logic - rename _backup_dir to backup_dir - add towerBackupClaim status to make the pvc name easier to find for users --- roles/backup/README.md | 2 +- roles/backup/tasks/awx-cro.yml | 2 +- roles/backup/tasks/main.yml | 2 +- roles/backup/tasks/postgres.yml | 10 ++-- roles/backup/tasks/secrets.yml | 8 +-- roles/backup/tasks/update_status.yml | 5 +- roles/backup/templates/event.yml.j2 | 4 +- roles/restore/README.md | 28 +++++++--- roles/restore/tasks/main.yml | 10 ++-- roles/restore/tasks/postgres.yml | 73 +++++++++++++-------------- roles/restore/tasks/update_status.yml | 4 +- roles/restore/templates/event.yml.j2 | 4 +- 12 files changed, 82 insertions(+), 70 deletions(-) diff --git a/roles/backup/README.md b/roles/backup/README.md index 8fa467d4..3d93035c 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -76,7 +76,7 @@ You can test this role directly by creating and running the following playbook w ``` --- -- name: Backup Tower +- name: Backup AWX hosts: localhost gather_facts: false roles: diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index a2b6df47..bcbb2bc5 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -35,4 +35,4 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ awx_object_template }}' > {{ _backup_dir }}/awx_object.yml" + bash -c "echo '{{ awx_object_template }}' > {{ backup_dir }}/awx_object.yml" diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index dfe9c05e..5fc6ab4d 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -24,7 +24,7 @@ - name: Set flag signifying this backup was successful set_fact: - tower_backup_complete: "{{ _backup_dir }}" + tower_backup_complete: true - include_tasks: cleanup.yml diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 2925be17..789c8d99 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -48,28 +48,28 @@ - name: Set backup directory name set_fact: - _backup_dir: "/backups/tower-openshift-backup-{{ now }}" + backup_dir: "/backups/tower-openshift-backup-{{ now }}" - name: Create directory for backup community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - mkdir -p {{ _backup_dir }} + mkdir -p {{ backup_dir }} - name: Precreate file for database dump community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - touch {{ _backup_dir }}/tower.db + touch {{ backup_dir }}/tower.db - name: Set permissions on file for database dump community.kubernetes.k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - chmod 0600 {{ _backup_dir }}/tower.db + chmod 0600 {{ backup_dir }}/tower.db - name: Set pg_dump command set_fact: @@ -85,5 +85,5 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" + bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ backup_dir }}/tower.db" register: data_migration diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index d5f0105e..5a276c0b 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -32,7 +32,7 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ secret_key_template }}' > {{ _backup_dir }}/secret_key_secret.yml" + bash -c "echo '{{ secret_key_template }}' > {{ backup_dir }}/secret_key_secret.yml" - name: Get admin_password k8s_info: @@ -60,7 +60,7 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ admin_password_template }}' > {{ _backup_dir }}/admin_password_secret.yml" + bash -c "echo '{{ admin_password_template }}' > {{ backup_dir }}/admin_password_secret.yml" - name: Get broadcast_websocket k8s_info: @@ -88,7 +88,7 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ broadcast_websocket_template }}' > {{ _backup_dir }}/broadcast_websocket_secret.yml" + bash -c "echo '{{ broadcast_websocket_template }}' > {{ backup_dir }}/broadcast_websocket_secret.yml" - name: Get postgres configuration k8s_info: @@ -121,4 +121,4 @@ namespace: "{{ meta.namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ postgres_secret_template }}' > {{ _backup_dir }}/postgres_secret.yml" + bash -c "echo '{{ postgres_secret_template }}' > {{ backup_dir }}/postgres_secret.yml" diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml index 1c6ef1c6..b2894f22 100644 --- a/roles/backup/tasks/update_status.yml +++ b/roles/backup/tasks/update_status.yml @@ -12,5 +12,6 @@ name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" status: - towerBackupComplete: "{{ _backup_dir }}" - when: tower_backup_complete is defined + towerBackupDirectory: "{{ backup_dir }}" + towerBackupClaim: "{{ backup_pvc }}" + when: tower_backup_complete diff --git a/roles/backup/templates/event.yml.j2 b/roles/backup/templates/event.yml.j2 index 3670cba3..ead6aea4 100644 --- a/roles/backup/templates/event.yml.j2 +++ b/roles/backup/templates/event.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Event metadata: - name: restore-error.{{ now }} + name: backup-error.{{ now }} namespace: {{ meta.namespace }} involvedObject: apiVersion: awx.ansible.com/v1beta1 @@ -10,7 +10,7 @@ involvedObject: name: {{ meta.name }} namespace: {{ meta.namespace }} message: {{ error_msg }} -reason: RestoreFailed +reason: BackupFailed type: Warning firstTimestamp: {{ now }} lastTimestamp: {{ now }} diff --git a/roles/restore/README.md b/roles/restore/README.md index d04c4d83..16e1d7ce 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -25,7 +25,7 @@ Then create a file named `restore-awx.yml` with the following contents: apiVersion: awx.ansible.com/v1beta1 kind: AWXRestore metadata: - name: awxrestore + name: restore1 namespace: my-namespace spec: tower_name: mytower @@ -33,8 +33,7 @@ spec: tower_backup_dir: tower-openshift-backup-2021-04-01-15:49:17 ``` -Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. - +Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. Finally, use `kubectl` to create the restore object in your cluster: @@ -48,7 +47,24 @@ This will create a new deployment and restore your backup to it. Role Variables -------------- -A custom, pre-created pvc can be used by setting the following variables. +The name of the backup directory can be found as a status on your AWXBackup object. This can be found in your cluster's console, or with the client as shown below. + +```bash +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.towerBackupDirectory}" +/backups/tower-openshift-backup-2021-04-02-03:25:08 +``` + +``` +tower_backup_dir: '/backups/tower-openshift-backup-2021-04-02-03:25:08' +``` + + +The name of the PVC can also be found by looking at the backup object. + +```bash +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.towerBackupClaim}" +awx-backup-volume-claim +``` ``` tower_backup_pvc: 'awx-backup-volume-claim' @@ -69,11 +85,11 @@ You can test this role directly by creating and running the following playbook w ``` --- -- name: Backup Tower +- name: Restore AWX hosts: localhost gather_facts: false roles: - - backup + - restore ``` License diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index df03f3d1..8b072d73 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -20,13 +20,13 @@ - include_tasks: init.yml - # - include_tasks: postgres.yml - # - # - include_tasks: secrets.yml + - include_tasks: postgres.yml - - name: Set flag signifying this backup was successful + - include_tasks: secrets.yml + + - name: Set flag signifying this restore was successful set_fact: - tower_backup_complete: True + tower_restore_complete: True - include_tasks: cleanup.yml diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index e6d375e3..44acf750 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -42,44 +42,39 @@ set_fact: postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" +- name: Check for presence of AWX Deployment + k8s_info: + api_version: v1 + kind: Deployment + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + register: this_deployment -# TODO: Add postgres restore logic, just pipe the pg_dump from PVC to db via psql +- name: Scale down Deployment for migration + k8s_scale: + api_version: v1 + kind: Deployment + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + replicas: 0 + when: this_deployment['resources'] | length -# -# - name: Create directory for backup -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# mkdir -p {{ _backup_dir }} -# -# - name: Precreate file for database dump -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# touch {{ _backup_dir }}/tower.db -# -# - name: Set permissions on file for database dump -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# chmod 0600 {{ _backup_dir }}/tower.db -# -# - name: Set pg_dump command -# set_fact: -# pgdump: >- -# pg_dump --clean --create -# -h {{ awx_postgres_host }} -# -U {{ awx_postgres_user }} -# -d {{ awx_postgres_database }} -# -p {{ awx_postgres_port }} -# -# - name: Write pg_dump to backup on PVC -# community.kubernetes.k8s_exec: -# namespace: "{{ meta.namespace }}" -# pod: "{{ meta.name }}-db-management" -# command: >- -# bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db" -# register: data_migration +- name: Set pg_restore command + set_fact: + psql_restore: >- + psql -U {{ awx_postgres_user }} + -d template1 + -p {{ awx_postgres_port }} + +- name: Restore database dump to the new postgresql container + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ postgres_pod_name }}" + command: | + bash -c """ + set -e -o pipefail + cat {{ tower_backup_dir }}/tower.db | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} + echo 'Successful' + """ + register: data_migration + failed_when: "'Successful' not in data_migration.stdout" diff --git a/roles/restore/tasks/update_status.yml b/roles/restore/tasks/update_status.yml index f056a1ac..92196d01 100644 --- a/roles/restore/tasks/update_status.yml +++ b/roles/restore/tasks/update_status.yml @@ -4,12 +4,12 @@ api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' -- name: Update Tower Backup status +- name: Update Tower Restore status operator_sdk.util.k8s_status: api_version: '{{ api_version }}' kind: "{{ kind }}" name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" status: - towerRestoreComplete: True + towerRestoreComplete: true when: tower_restore_complete is defined diff --git a/roles/restore/templates/event.yml.j2 b/roles/restore/templates/event.yml.j2 index ead6aea4..3670cba3 100644 --- a/roles/restore/templates/event.yml.j2 +++ b/roles/restore/templates/event.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: Event metadata: - name: backup-error.{{ now }} + name: restore-error.{{ now }} namespace: {{ meta.namespace }} involvedObject: apiVersion: awx.ansible.com/v1beta1 @@ -10,7 +10,7 @@ involvedObject: name: {{ meta.name }} namespace: {{ meta.namespace }} message: {{ error_msg }} -reason: BackupFailed +reason: RestoreFailed type: Warning firstTimestamp: {{ now }} lastTimestamp: {{ now }} From 5669747bbfbdd954ce90934217c2a2f514df8e72 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 2 Apr 2021 01:36:56 -0400 Subject: [PATCH 20/43] Scope pvc and management pod to default namespace - make this configurable via tower_backup_pvc_namespace var - remove redundant k8s task info --- ansible/templates/awxbackup_crd.yml.j2 | 3 +++ ansible/templates/awxrestore_crd.yml.j2 | 3 +++ deploy/awx-operator.yaml | 8 ++++++++ deploy/crds/awxbackup_v1beta1_crd.yaml | 3 +++ deploy/crds/awxrestore_v1beta1_crd.yaml | 3 +++ roles/backup/defaults/main.yml | 1 + roles/backup/tasks/cleanup.yml | 2 +- roles/backup/tasks/init.yml | 8 +++----- roles/backup/templates/backup_pvc.yml.j2 | 2 +- roles/backup/templates/management-pod.yml.j2 | 2 +- roles/backup/vars/main.yml | 1 - roles/restore/README.md | 4 ++-- roles/restore/defaults/main.yml | 1 + roles/restore/tasks/cleanup.yml | 2 +- roles/restore/tasks/init.yml | 5 ++--- roles/restore/tasks/preflight.yml | 8 +++++++- roles/restore/templates/management-pod.yml.j2 | 2 +- 17 files changed, 41 insertions(+), 17 deletions(-) diff --git a/ansible/templates/awxbackup_crd.yml.j2 b/ansible/templates/awxbackup_crd.yml.j2 index d0ad49a9..02e18a27 100644 --- a/ansible/templates/awxbackup_crd.yml.j2 +++ b/ansible/templates/awxbackup_crd.yml.j2 @@ -32,6 +32,9 @@ spec: tower_backup_pvc: description: Name of the PVC to be used for storing the backup type: string + tower_backup_pvc_namespace: + description: Namespace PVC is in + type: string tower_backup_size: description: Size of PVC type: string diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 index 39b669a6..e7e65923 100644 --- a/ansible/templates/awxrestore_crd.yml.j2 +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -32,6 +32,9 @@ spec: tower_backup_pvc: description: Name of the PVC to be used for storing the backup type: string + tower_backup_pvc_namespace: + description: Namespace PVC is in + type: string tower_backup_dir: description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) type: string diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 6fc80e38..109337a2 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -560,6 +560,10 @@ spec: tower_backup_pvc: description: Name of the PVC to be used for storing the backup type: string + tower_backup_pvc_namespace: + default: 'default' + description: Namespace PVC is in + type: string tower_backup_size: description: Size of PVC type: string @@ -615,6 +619,10 @@ spec: tower_backup_pvc: description: Name of the PVC to be used for storing the backup type: string + tower_backup_pvc_namespace: + default: 'default' + description: Namespace PVC is in + type: string tower_backup_size: description: Size of PVC type: string diff --git a/deploy/crds/awxbackup_v1beta1_crd.yaml b/deploy/crds/awxbackup_v1beta1_crd.yaml index d0ad49a9..02e18a27 100644 --- a/deploy/crds/awxbackup_v1beta1_crd.yaml +++ b/deploy/crds/awxbackup_v1beta1_crd.yaml @@ -32,6 +32,9 @@ spec: tower_backup_pvc: description: Name of the PVC to be used for storing the backup type: string + tower_backup_pvc_namespace: + description: Namespace PVC is in + type: string tower_backup_size: description: Size of PVC type: string diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml index 39b669a6..e7e65923 100644 --- a/deploy/crds/awxrestore_v1beta1_crd.yaml +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -32,6 +32,9 @@ spec: tower_backup_pvc: description: Name of the PVC to be used for storing the backup type: string + tower_backup_pvc_namespace: + description: Namespace PVC is in + type: string tower_backup_dir: description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) type: string diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 59310cdd..2076c68c 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -4,6 +4,7 @@ tower_name: '' # Specify a pre-created PVC (name) to backup to tower_backup_pvc: '' +tower_backup_pvc_namespace: 'default' # Size of backup PVC if created dynamically tower_backup_size: '' diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml index e20e8718..11dc6d07 100644 --- a/roles/backup/tasks/cleanup.yml +++ b/roles/backup/tasks/cleanup.yml @@ -11,6 +11,6 @@ community.kubernetes.k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" state: absent force: true diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 68fb305c..27158bf6 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -4,7 +4,7 @@ community.kubernetes.k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" state: absent force: true wait: true @@ -14,7 +14,7 @@ k8s_info: name: "{{ tower_backup_pvc }}" kind: PersistentVolumeClaim - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" register: provided_pvc when: - tower_backup_pvc != '' @@ -48,16 +48,14 @@ - name: Create PVC for backup community.kubernetes.k8s: kind: PersistentVolumeClaim - namespace: "{{ meta.namespace }}" template: "backup_pvc.yml.j2" when: - tower_backup_pvc == '' or tower_backup_pvc is not defined - name: Create management pod from templated deployment config community.kubernetes.k8s: - name: "{{ meta.name }}-db-management" + name: "{{ tower_backup_pvc_namespace }}-db-management" kind: Deployment - namespace: "{{ meta.namespace }}" state: present template: "management-pod.yml.j2" wait: true diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index 693b00ec..57778b82 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: {{ meta.name }}-backup-claim - namespace: {{ meta.namespace}} + namespace: {{ tower_backup_pvc_namespace }} spec: accessModes: - ReadWriteOnce diff --git a/roles/backup/templates/management-pod.yml.j2 b/roles/backup/templates/management-pod.yml.j2 index 87ddff8e..d938da20 100644 --- a/roles/backup/templates/management-pod.yml.j2 +++ b/roles/backup/templates/management-pod.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Pod metadata: name: {{ meta.name }}-db-management - namespace: {{ meta.namespace }} + namespace: {{ tower_backup_pvc_namespace }} spec: containers: - name: {{ meta.name }}-db-management diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index 851b98f0..dbb48bf8 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -1,4 +1,3 @@ --- - deployment_type: "awx" tower_postgres_image: postgres:12 diff --git a/roles/restore/README.md b/roles/restore/README.md index 16e1d7ce..1325c402 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -29,8 +29,8 @@ metadata: namespace: my-namespace spec: tower_name: mytower - tower_backup_pvc: myoldtower-awxbackup-adfx7ciow - tower_backup_dir: tower-openshift-backup-2021-04-01-15:49:17 + tower_backup_pvc: awxbackup1-backup-claim + tower_backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 ``` Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml index 67a38b79..3bafde7e 100644 --- a/roles/restore/defaults/main.yml +++ b/roles/restore/defaults/main.yml @@ -4,6 +4,7 @@ tower_name: '' # Required: specify a pre-created PVC (name) to restore from tower_backup_pvc: '' +tower_backup_pvc_namespace: 'default' # Required: backup name, found on the awxbackup object tower_backup_dir: '' diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml index 9976a8c9..7b094f32 100644 --- a/roles/restore/tasks/cleanup.yml +++ b/roles/restore/tasks/cleanup.yml @@ -4,6 +4,6 @@ community.kubernetes.k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" state: absent force: true diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index 009fba00..f650a568 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -4,7 +4,7 @@ community.kubernetes.k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" state: absent force: true wait: true @@ -14,7 +14,7 @@ k8s_info: name: "{{ tower_backup_pvc }}" kind: PersistentVolumeClaim - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" register: provided_pvc when: - tower_backup_pvc != '' @@ -49,7 +49,6 @@ community.kubernetes.k8s: name: "{{ meta.name }}-db-management" kind: Deployment - namespace: "{{ meta.namespace }}" state: present template: "management-pod.yml.j2" wait: true diff --git a/roles/restore/tasks/preflight.yml b/roles/restore/tasks/preflight.yml index 108162ec..d9c98553 100644 --- a/roles/restore/tasks/preflight.yml +++ b/roles/restore/tasks/preflight.yml @@ -1,11 +1,17 @@ --- +- name: Create namespace for deployment + k8s: + name: "{{ meta.namespace }}" + kind: Namespace + state: present + # Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. - name: Check provided PVC exists k8s_info: name: "{{ tower_backup_pvc }}" kind: PersistentVolumeClaim - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" register: provided_pvc when: - tower_backup_pvc != '' diff --git a/roles/restore/templates/management-pod.yml.j2 b/roles/restore/templates/management-pod.yml.j2 index 87ddff8e..d938da20 100644 --- a/roles/restore/templates/management-pod.yml.j2 +++ b/roles/restore/templates/management-pod.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Pod metadata: name: {{ meta.name }}-db-management - namespace: {{ meta.namespace }} + namespace: {{ tower_backup_pvc_namespace }} spec: containers: - name: {{ meta.name }}-db-management From b9d0852c83de0cfc5151c5f96b6ba7870859fe61 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Mon, 5 Apr 2021 17:57:12 -0400 Subject: [PATCH 21/43] Fix small namespace issue --- ansible/instantiate-awx-deployment.yml | 2 +- roles/backup/tasks/awx-cro.yml | 2 +- roles/backup/tasks/init.yml | 2 +- roles/backup/tasks/main.yml | 2 +- roles/backup/tasks/postgres.yml | 8 ++++---- roles/backup/tasks/secrets.yml | 10 +++++----- roles/backup/vars/main.yml | 1 + roles/installer/tasks/migrate_data.yml | 2 +- roles/restore/tasks/init_awx.yml | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ansible/instantiate-awx-deployment.yml b/ansible/instantiate-awx-deployment.yml index 3c7b761c..0088a430 100644 --- a/ansible/instantiate-awx-deployment.yml +++ b/ansible/instantiate-awx-deployment.yml @@ -9,7 +9,7 @@ - name: Deploy AWX k8s: state: "{{ state | default('present') }}" - namespace: "{{ tower_namespace | default('default') }}" + namespace: "{{ namespace | default('default') }}" apply: yes wait: yes definition: diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index bcbb2bc5..2c5f93ac 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -32,7 +32,7 @@ - name: Write awx object to pvc community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "echo '{{ awx_object_template }}' > {{ backup_dir }}/awx_object.yml" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 27158bf6..f72e4006 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -54,7 +54,7 @@ - name: Create management pod from templated deployment config community.kubernetes.k8s: - name: "{{ tower_backup_pvc_namespace }}-db-management" + name: "{{ meta.name }}-db-management" kind: Deployment state: present template: "management-pod.yml.j2" diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 5fc6ab4d..e682e860 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -29,7 +29,7 @@ - include_tasks: cleanup.yml when: - - this_backup['resources'][0]['status']['towerBackupComplete'] is not defined + - this_backup['resources'][0]['status']['towerBackupDirectory'] is not defined - name: Update status variables include_tasks: update_status.yml diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 789c8d99..80a933a2 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -52,21 +52,21 @@ - name: Create directory for backup community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- mkdir -p {{ backup_dir }} - name: Precreate file for database dump community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- touch {{ backup_dir }}/tower.db - name: Set permissions on file for database dump community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- chmod 0600 {{ backup_dir }}/tower.db @@ -82,7 +82,7 @@ - name: Write pg_dump to backup on PVC community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ backup_dir }}/tower.db" diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 5a276c0b..2e62a88f 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -29,7 +29,7 @@ - name: Write secret_key to pvc community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "echo '{{ secret_key_template }}' > {{ backup_dir }}/secret_key_secret.yml" @@ -57,7 +57,7 @@ - name: Write secret_key to pvc community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "echo '{{ admin_password_template }}' > {{ backup_dir }}/admin_password_secret.yml" @@ -85,7 +85,7 @@ - name: Write broadcast_websocket definition to pvc community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "echo '{{ broadcast_websocket_template }}' > {{ backup_dir }}/broadcast_websocket_secret.yml" @@ -93,7 +93,7 @@ - name: Get postgres configuration k8s_info: kind: Secret - namespace: '{{ meta.namespace }}' + namespace: '{{ tower_backup_pvc_namespace }}' name: '{{ tower_postgres_configuration_secret }}' register: _postgres_configuration @@ -118,7 +118,7 @@ - name: Write postgres configuration to pvc community.kubernetes.k8s_exec: - namespace: "{{ meta.namespace }}" + namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "echo '{{ postgres_secret_template }}' > {{ backup_dir }}/postgres_secret.yml" diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index dbb48bf8..f1dad4a2 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -1,3 +1,4 @@ --- deployment_type: "awx" tower_postgres_image: postgres:12 +tower_backup_complete: false diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index 698ba081..e49b6e91 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -58,4 +58,4 @@ - name: Set flag signifying that this instance has been migrated set_fact: - tower_migrated_from_secret: "{{ awx_old_postgres_configuration_secret }}" + tower_migrated_from_secret: "{{ tower_old_postgres_configuration_secret }}" diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/init_awx.yml index 1e8c268c..be72f5cb 100644 --- a/roles/restore/tasks/init_awx.yml +++ b/roles/restore/tasks/init_awx.yml @@ -2,7 +2,7 @@ - name: Deploy AWX k8s: state: "{{ state | default('present') }}" - namespace: "{{ tower_namespace | default('default') }}" + namespace: "{{ namespace | default('default') }}" apply: yes wait: yes template: "{{ tower_backup_dir }}/awx_object.yml" From ce8c58f542689db52e54162ab77b29aa0e8aeaa4 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 8 Apr 2021 10:01:17 -0400 Subject: [PATCH 22/43] added secrets logic, fixed permissions issues --- roles/backup/tasks/cleanup.yml | 3 +- roles/backup/tasks/postgres.yml | 2 +- roles/backup/tasks/secrets.yml | 8 ++-- roles/restore/README.md | 8 +++- roles/restore/defaults/main.yml | 1 + roles/restore/tasks/apply_secret.yml | 24 ++++++++++ roles/restore/tasks/cleanup.yml | 5 +++ roles/restore/tasks/init.yml | 67 ++++++++++++++++++++-------- roles/restore/tasks/init_awx.yml | 22 ++++++++- roles/restore/tasks/init_secrets.yml | 9 ++++ roles/restore/tasks/main.yml | 4 +- roles/restore/tasks/preflight.yml | 33 -------------- 12 files changed, 123 insertions(+), 63 deletions(-) create mode 100644 roles/restore/tasks/apply_secret.yml create mode 100644 roles/restore/tasks/init_secrets.yml delete mode 100644 roles/restore/tasks/preflight.yml diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml index 11dc6d07..3cfc89c9 100644 --- a/roles/backup/tasks/cleanup.yml +++ b/roles/backup/tasks/cleanup.yml @@ -3,10 +3,9 @@ # After copying secret files to the PVC, delete the local tmp copies - name: Clean up _secrets directory ansible.builtin.file: - path: "{{ playbook_dir }}/_secrets" + path: "_secrets" state: absent - - name: Delete any existing management pod community.kubernetes.k8s: name: "{{ meta.name }}-db-management" diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 80a933a2..a8eb5ec5 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -69,7 +69,7 @@ namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - chmod 0600 {{ backup_dir }}/tower.db + chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db - name: Set pg_dump command set_fact: diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 2e62a88f..c7b8a520 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -21,7 +21,7 @@ template: src: secret_key_secret.yml.j2 dest: "_secrets/secret_key_secret.yml" - mode: '0600' + mode: '0700' - name: Set secret key template set_fact: @@ -49,7 +49,7 @@ template: src: admin_password_secret.yml.j2 dest: "_secrets/admin_password_secret.yml" - mode: '0600' + mode: '0700' - name: Set admin_password template set_fact: @@ -77,7 +77,7 @@ template: src: broadcast_websocket_secret.yml.j2 dest: "_secrets/broadcast_websocket_secret.yml" - mode: '0600' + mode: '0700' - name: Set broadcast_websocket template set_fact: @@ -110,7 +110,7 @@ template: src: postgres_secret.yml.j2 dest: "_secrets/postgres_secret.yml" - mode: '0600' + mode: '0700' - name: Set postgres configuration set_fact: diff --git a/roles/restore/README.md b/roles/restore/README.md index 1325c402..ae262c11 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -33,7 +33,13 @@ spec: tower_backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 ``` -Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. +Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. + +The namespace specified is the namespace the resulting AWX deployment will be in. The namespace you specified must be pre-created. + +``` +kubectl create ns my-namespace +``` Finally, use `kubectl` to create the restore object in your cluster: diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml index 3bafde7e..29a91b76 100644 --- a/roles/restore/defaults/main.yml +++ b/roles/restore/defaults/main.yml @@ -6,6 +6,7 @@ tower_name: '' tower_backup_pvc: '' tower_backup_pvc_namespace: 'default' +# TODO: If the backup_dir is not provided, it should default to the most recent backup based on the timestamp at the end of the file name. # Required: backup name, found on the awxbackup object tower_backup_dir: '' diff --git a/roles/restore/tasks/apply_secret.yml b/roles/restore/tasks/apply_secret.yml new file mode 100644 index 00000000..69e87505 --- /dev/null +++ b/roles/restore/tasks/apply_secret.yml @@ -0,0 +1,24 @@ +--- + +- name: Get secret definition from pvc + community.kubernetes.k8s_exec: + namespace: "{{ tower_backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "cat '{{ tower_backup_dir }}/{{ item }}.yml'" + register: awx_object + +- name: Write temp secret definition template file + copy: + dest: "_definitions/{{ item }}.yml.j2" + content: | + {{ awx_object.stdout }} + mode: '0600' + +- name: Apply secret + k8s: + state: "{{ state | default('present') }}" + namespace: "{{ namespace | default('default') }}" + apply: yes + wait: yes + template: "_definitions/{{ item }}.yml.j2" diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml index 7b094f32..e9ea732e 100644 --- a/roles/restore/tasks/cleanup.yml +++ b/roles/restore/tasks/cleanup.yml @@ -1,5 +1,10 @@ --- +- name: Clean up _secrets directory + ansible.builtin.file: + path: "_definitions" + state: absent + - name: Delete any existing management pod community.kubernetes.k8s: name: "{{ meta.name }}-db-management" diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index f650a568..c6a500bb 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -1,13 +1,12 @@ --- +- name: Set default pvc name + set_fact: + _default_backup_pvc: "{{ meta.name }}-backup-claim" -- name: Delete any existing management pod - community.kubernetes.k8s: - name: "{{ meta.name }}-db-management" - kind: Pod - namespace: "{{ tower_backup_pvc_namespace }}" - state: absent - force: true - wait: true +# by default, it will re-use the old pvc if already created (unless pvc is provided) +- name: Set PVC to use for backup + set_fact: + backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" # Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. - name: Check provided PVC exists @@ -30,20 +29,19 @@ - name: Fail early if pvc is defined but does not exist fail: - msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + msg: "{{ error_msg }}" when: - tower_backup_pvc != '' - provided_pvc.resources | length == 0 -# If tower_backup_pvc is defined, use in management-pod.yml.j2 -- name: Set default pvc name - set_fact: - _default_backup_pvc: "{{ meta.name }}-backup-claim" - -# by default, it will re-use the old pvc if already created (unless pvc is provided) -- name: Set PVC to use for backup - set_fact: - backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ tower_backup_pvc_namespace }}" + state: absent + force: true + wait: true - name: Create management pod from templated deployment config community.kubernetes.k8s: @@ -52,3 +50,36 @@ state: present template: "management-pod.yml.j2" wait: true + +- name: Check to make sure backup directory exists on PVC + community.kubernetes.k8s_exec: + namespace: "{{ tower_backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "stat {{ tower_backup_dir }}" + register: stat_backup_dir + +- debug: + msg: "{{stat_backup_dir}}" + +- name: Error if backup dir is missing + block: + - name: Set error message + set_fact: + error_msg: "{{ tower_backup_dir }} does not exist, see the towerBackupDirectory status on your AWXBackup for the correct tower_backup_dir." + + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if backup dir provided does not exist + fail: + msg: "{{ error_msg }}" + when: + - tower_backup_dir != '' + - stat_backup_dir.return_code != 0 + +- name: Make _definitions directory + file: + path: "_definitions" + state: directory + mode: '0700' diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/init_awx.yml index be72f5cb..47c74322 100644 --- a/roles/restore/tasks/init_awx.yml +++ b/roles/restore/tasks/init_awx.yml @@ -1,11 +1,31 @@ --- + +- name: Get AWX object definition from pvc + community.kubernetes.k8s_exec: + namespace: "{{ tower_backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "cat '{{ tower_backup_dir }}/awx_object.yml'" + register: awx_object + +- name: Write temp AWX definition template file + copy: + dest: "_definitions/awx_object.yml.j2" + content: | + {{ awx_object.stdout }} + mode: '0600' + - name: Deploy AWX k8s: state: "{{ state | default('present') }}" namespace: "{{ namespace | default('default') }}" apply: yes wait: yes - template: "{{ tower_backup_dir }}/awx_object.yml" + template: "_definitions/awx_object.yml.j2" + wait: true + + +# TODO: The awx object and secrets need to be applied from the awx-operator, because that is where the service account is?. So we will need to either copy them over or pipe them into a template command # TODO: Add logic to allow users to provide override values here, # or to specify spec values that were not in the backed up AWX object. diff --git a/roles/restore/tasks/init_secrets.yml b/roles/restore/tasks/init_secrets.yml new file mode 100644 index 00000000..dab44259 --- /dev/null +++ b/roles/restore/tasks/init_secrets.yml @@ -0,0 +1,9 @@ +--- + +- name: Get secret definition from pvc + include_tasks: apply_secrets.yml + with_items: + - secret_key_secret + - admin_password_secret + - broadcast_websocket_secret + - postgres_secret diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index 8b072d73..63d1138d 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -14,12 +14,10 @@ register: this_restore - block: - - include_tasks: preflight.yml + - include_tasks: init.yml - include_tasks: init_awx.yml - - include_tasks: init.yml - - include_tasks: postgres.yml - include_tasks: secrets.yml diff --git a/roles/restore/tasks/preflight.yml b/roles/restore/tasks/preflight.yml deleted file mode 100644 index d9c98553..00000000 --- a/roles/restore/tasks/preflight.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: Create namespace for deployment - k8s: - name: "{{ meta.namespace }}" - kind: Namespace - state: present - -# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. -- name: Check provided PVC exists - k8s_info: - name: "{{ tower_backup_pvc }}" - kind: PersistentVolumeClaim - namespace: "{{ tower_backup_pvc_namespace }}" - register: provided_pvc - when: - - tower_backup_pvc != '' - -- name: Surface error to user - block: - - name: Set error message - set_fact: - error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." - - - name: Handle error - import_tasks: error_handling.yml - - - name: Fail early if pvc is defined but does not exist - fail: - msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." - when: - - tower_backup_pvc != '' - - provided_pvc.resources | length == 0 From 2cbf60fa1751e5b902a6a761f45fab9943a31b81 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 8 Apr 2021 11:20:56 -0400 Subject: [PATCH 23/43] Remove unneeded fqcn for modules & fix CI --- ansible/templates/awx-operator.yaml.j2 | 10 +- deploy/awx-operator.yaml | 323 +++++++++--------- molecule/default/prepare.yml | 2 +- roles/backup/tasks/awx-cro.yml | 2 +- roles/backup/tasks/cleanup.yml | 4 +- roles/backup/tasks/error_handling.yml | 2 +- roles/backup/tasks/init.yml | 6 +- roles/backup/tasks/postgres.yml | 8 +- roles/backup/tasks/secrets.yml | 8 +- roles/installer/tasks/initialize_django.yml | 6 +- .../tasks/load_ldap_cacert_secret.yml | 2 +- .../installer/tasks/load_route_tls_secret.yml | 2 +- roles/installer/tasks/migrate_data.yml | 2 +- roles/installer/tasks/update_status.yml | 4 +- roles/restore/tasks/apply_secret.yml | 2 +- roles/restore/tasks/cleanup.yml | 4 +- roles/restore/tasks/error_handling.yml | 2 +- roles/restore/tasks/init.yml | 9 +- roles/restore/tasks/init_awx.yml | 6 +- roles/restore/tasks/postgres.yml | 2 +- 20 files changed, 198 insertions(+), 208 deletions(-) diff --git a/ansible/templates/awx-operator.yaml.j2 b/ansible/templates/awx-operator.yaml.j2 index 31fe55b6..21a74d67 100644 --- a/ansible/templates/awx-operator.yaml.j2 +++ b/ansible/templates/awx-operator.yaml.j2 @@ -3,6 +3,10 @@ # Update templates under ansible/templates/ {% include 'crd.yml.j2' %} +{% include 'awxbackup_crd.yml.j2' %} + +{% include 'awxrestore_crd.yml.j2' %} + {% include 'role.yml.j2' %} {% include 'role_binding.yml.j2' %} @@ -10,9 +14,3 @@ {% include 'service_account.yml.j2' %} {% include 'operator.yml.j2' %} - -{% include 'crd.yml.j2' %} - -{% include 'awxbackup_crd.yml.j2' %} - -{% include 'awxbackup_crd.yml.j2' %} diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 109337a2..d5119824 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -1,155 +1,5 @@ # This file is generated by Ansible. Changes will be lost. # Update templates under ansible/templates/ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: awx-operator -rules: - - apiGroups: - - route.openshift.io - resources: - - routes - - routes/custom-host - verbs: - - '*' - - apiGroups: - - "" - - "rbac.authorization.k8s.io" - resources: - - pods - - services - - services/finalizers - - serviceaccounts - - endpoints - - persistentvolumeclaims - - events - - configmaps - - secrets - - roles - - rolebindings - verbs: - - '*' - - apiGroups: - - apps - - extensions - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - - ingresses - verbs: - - '*' - - apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create - - apiGroups: - - apps - resourceNames: - - awx-operator - resources: - - deployments/finalizers - verbs: - - update - - apiGroups: - - apps - resources: - - deployments/scale - verbs: - - patch - - apiGroups: - - "" - resources: - - pods/exec - verbs: - - create - - get - - apiGroups: - - apps - resources: - - replicasets - verbs: - - get - - apiGroups: - - awx.ansible.com - resources: - - '*' - - awxbackups - verbs: - - '*' - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: awx-operator -subjects: - - kind: ServiceAccount - name: awx-operator - namespace: default -roleRef: - kind: ClusterRole - name: awx-operator - apiGroup: rbac.authorization.k8s.io - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: awx-operator - namespace: default - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: awx-operator -spec: - replicas: 1 - selector: - matchLabels: - name: awx-operator - template: - metadata: - labels: - name: awx-operator - spec: - serviceAccountName: awx-operator - containers: - - name: awx-operator - image: "quay.io/ansible/awx-operator:0.7.0" - imagePullPolicy: "Always" - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - env: - # Watch all namespaces (cluster-scoped). - - name: WATCH_NAMESPACE - value: "" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: awx-operator - - name: ANSIBLE_GATHERING - value: explicit - livenessProbe: - httpGet: - path: /healthz - port: 6789 - initialDelaySeconds: 15 - periodSeconds: 20 - volumes: - - name: runner - emptyDir: {} - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -561,7 +411,6 @@ spec: description: Name of the PVC to be used for storing the backup type: string tower_backup_pvc_namespace: - default: 'default' description: Namespace PVC is in type: string tower_backup_size: @@ -589,14 +438,14 @@ spec: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: awxbackups.awx.ansible.com + name: awxrestores.awx.ansible.com spec: group: awx.ansible.com names: - kind: AWXBackup - listKind: AWXBackupList - plural: awxbackups - singular: awxbackup + kind: AWXRestore + listKind: AWXRestoreList + plural: awxrestores + singular: awxrestore scope: Namespaced versions: - name: v1beta1 @@ -614,20 +463,16 @@ spec: type: object properties: tower_name: - description: Name of the deployment to be backed up + description: Name of the deployment to be restored to type: string tower_backup_pvc: description: Name of the PVC to be used for storing the backup type: string tower_backup_pvc_namespace: - default: 'default' description: Namespace PVC is in type: string - tower_backup_size: - description: Size of PVC - type: string - tower_backup_storage_class: - description: Storage class to use when creating PVC for backup + tower_backup_dir: + description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) type: string tower_secret_key_secret: description: Custom secret_key secret name @@ -642,4 +487,154 @@ spec: description: Custom postgres_configuration secret name type: string oneOf: - - required: ["tower_name"] + - required: ["tower_name", "tower_backup_pvc"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: awx-operator +rules: + - apiGroups: + - route.openshift.io + resources: + - routes + - routes/custom-host + verbs: + - '*' + - apiGroups: + - "" + - "rbac.authorization.k8s.io" + resources: + - pods + - services + - services/finalizers + - serviceaccounts + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + - roles + - rolebindings + verbs: + - '*' + - apiGroups: + - apps + - extensions + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + - ingresses + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - apps + resourceNames: + - awx-operator + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - apps + resources: + - deployments/scale + verbs: + - patch + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - apiGroups: + - awx.ansible.com + resources: + - '*' + - awxbackups + verbs: + - '*' + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: awx-operator +subjects: + - kind: ServiceAccount + name: awx-operator + namespace: default +roleRef: + kind: ClusterRole + name: awx-operator + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: awx-operator + namespace: default + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: awx-operator +spec: + replicas: 1 + selector: + matchLabels: + name: awx-operator + template: + metadata: + labels: + name: awx-operator + spec: + serviceAccountName: awx-operator + containers: + - name: awx-operator + image: "quay.io/chadams/awx-operator:new-crd" + imagePullPolicy: "Always" + volumeMounts: + - mountPath: /tmp/ansible-operator/runner + name: runner + env: + # Watch all namespaces (cluster-scoped). + - name: WATCH_NAMESPACE + value: "" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: awx-operator + - name: ANSIBLE_GATHERING + value: explicit + livenessProbe: + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - name: runner + emptyDir: {} diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index 76ce4ad0..118388ea 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -19,7 +19,7 @@ k8s: definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxbackup_v1beta1_crd.yaml'])) }}" - - name: Create AWXBackup Custom Resource Definition + - name: Create AWXRestore Custom Resource Definition k8s: definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxrestore_v1beta1_crd.yaml'])) }}" diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index 2c5f93ac..4db2f218 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -31,7 +31,7 @@ awx_object_template: "{{ lookup('file', '_secrets/awx_object.yml') }}" - name: Write awx object to pvc - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml index 3cfc89c9..aa9c6b11 100644 --- a/roles/backup/tasks/cleanup.yml +++ b/roles/backup/tasks/cleanup.yml @@ -2,12 +2,12 @@ # After copying secret files to the PVC, delete the local tmp copies - name: Clean up _secrets directory - ansible.builtin.file: + file: path: "_secrets" state: absent - name: Delete any existing management pod - community.kubernetes.k8s: + k8s: name: "{{ meta.name }}-db-management" kind: Pod namespace: "{{ tower_backup_pvc_namespace }}" diff --git a/roles/backup/tasks/error_handling.yml b/roles/backup/tasks/error_handling.yml index f3361d6c..de1c28ce 100644 --- a/roles/backup/tasks/error_handling.yml +++ b/roles/backup/tasks/error_handling.yml @@ -10,7 +10,7 @@ now: '{{ lookup("pipe", "date +%FT%TZ") }}' - name: Emit ocp event with error - community.kubernetes.k8s: + k8s: kind: Event namespace: "{{ meta.namespace }}" template: "event.yml.j2" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index f72e4006..119c68e1 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -1,7 +1,7 @@ --- - name: Delete any existing management pod - community.kubernetes.k8s: + k8s: name: "{{ meta.name }}-db-management" kind: Pod namespace: "{{ tower_backup_pvc_namespace }}" @@ -46,14 +46,14 @@ backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" - name: Create PVC for backup - community.kubernetes.k8s: + k8s: kind: PersistentVolumeClaim template: "backup_pvc.yml.j2" when: - tower_backup_pvc == '' or tower_backup_pvc is not defined - name: Create management pod from templated deployment config - community.kubernetes.k8s: + k8s: name: "{{ meta.name }}-db-management" kind: Deployment state: present diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index a8eb5ec5..fa065780 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -51,21 +51,21 @@ backup_dir: "/backups/tower-openshift-backup-{{ now }}" - name: Create directory for backup - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- mkdir -p {{ backup_dir }} - name: Precreate file for database dump - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- touch {{ backup_dir }}/tower.db - name: Set permissions on file for database dump - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- @@ -81,7 +81,7 @@ -p {{ awx_postgres_port }} - name: Write pg_dump to backup on PVC - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index c7b8a520..bab87c52 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -28,7 +28,7 @@ secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml') }}" - name: Write secret_key to pvc - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- @@ -56,7 +56,7 @@ admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml') }}" - name: Write secret_key to pvc - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- @@ -84,7 +84,7 @@ broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml') }}" - name: Write broadcast_websocket definition to pvc - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- @@ -117,7 +117,7 @@ postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml') }}" - name: Write postgres configuration to pvc - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- diff --git a/roles/installer/tasks/initialize_django.yml b/roles/installer/tasks/initialize_django.yml index 62849d7e..cb6a6e1b 100644 --- a/roles/installer/tasks/initialize_django.yml +++ b/roles/installer/tasks/initialize_django.yml @@ -1,6 +1,6 @@ --- - name: Check if there are any super users defined. - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" @@ -14,7 +14,7 @@ changed_when: users_result.return_code > 0 - name: Create super user via Django if it doesn't exist. - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" @@ -25,7 +25,7 @@ when: users_result.return_code > 0 - name: Create preload data if necessary. # noqa 305 - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" diff --git a/roles/installer/tasks/load_ldap_cacert_secret.yml b/roles/installer/tasks/load_ldap_cacert_secret.yml index 41667a1b..ebf5fcc2 100644 --- a/roles/installer/tasks/load_ldap_cacert_secret.yml +++ b/roles/installer/tasks/load_ldap_cacert_secret.yml @@ -1,6 +1,6 @@ --- - name: Retrieve LDAP CA Certificate Secret - community.kubernetes.k8s_info: + k8s_info: kind: Secret namespace: '{{ meta.namespace }}' name: '{{ ldap_cacert_secret }}' diff --git a/roles/installer/tasks/load_route_tls_secret.yml b/roles/installer/tasks/load_route_tls_secret.yml index 03b50226..529e5851 100644 --- a/roles/installer/tasks/load_route_tls_secret.yml +++ b/roles/installer/tasks/load_route_tls_secret.yml @@ -1,6 +1,6 @@ --- - name: Retrieve Route TLS Secret - community.kubernetes.k8s_info: + k8s_info: kind: Secret namespace: '{{ meta.namespace }}' name: '{{ tower_route_tls_secret }}' diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index e49b6e91..28604306 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -44,7 +44,7 @@ -p {{ awx_postgres_port }} - name: Stream backup from pg_dump to the new postgresql container - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ postgres_pod_name }}" command: | diff --git a/roles/installer/tasks/update_status.yml b/roles/installer/tasks/update_status.yml index ec4b3d54..0b123d9a 100644 --- a/roles/installer/tasks/update_status.yml +++ b/roles/installer/tasks/update_status.yml @@ -18,7 +18,7 @@ towerAdminUser: "{{ tower_admin_user }}" - name: Retrieve instance version - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" @@ -47,7 +47,7 @@ - block: - name: Retrieve route URL - community.kubernetes.k8s_info: + k8s_info: kind: Route namespace: '{{ meta.namespace }}' name: '{{ meta.name }}' diff --git a/roles/restore/tasks/apply_secret.yml b/roles/restore/tasks/apply_secret.yml index 69e87505..095ac470 100644 --- a/roles/restore/tasks/apply_secret.yml +++ b/roles/restore/tasks/apply_secret.yml @@ -1,7 +1,7 @@ --- - name: Get secret definition from pvc - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml index e9ea732e..6706916c 100644 --- a/roles/restore/tasks/cleanup.yml +++ b/roles/restore/tasks/cleanup.yml @@ -1,12 +1,12 @@ --- - name: Clean up _secrets directory - ansible.builtin.file: + file: path: "_definitions" state: absent - name: Delete any existing management pod - community.kubernetes.k8s: + k8s: name: "{{ meta.name }}-db-management" kind: Pod namespace: "{{ tower_backup_pvc_namespace }}" diff --git a/roles/restore/tasks/error_handling.yml b/roles/restore/tasks/error_handling.yml index f3361d6c..de1c28ce 100644 --- a/roles/restore/tasks/error_handling.yml +++ b/roles/restore/tasks/error_handling.yml @@ -10,7 +10,7 @@ now: '{{ lookup("pipe", "date +%FT%TZ") }}' - name: Emit ocp event with error - community.kubernetes.k8s: + k8s: kind: Event namespace: "{{ meta.namespace }}" template: "event.yml.j2" diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index c6a500bb..3a2280c8 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -35,7 +35,7 @@ - provided_pvc.resources | length == 0 - name: Delete any existing management pod - community.kubernetes.k8s: + k8s: name: "{{ meta.name }}-db-management" kind: Pod namespace: "{{ tower_backup_pvc_namespace }}" @@ -44,7 +44,7 @@ wait: true - name: Create management pod from templated deployment config - community.kubernetes.k8s: + k8s: name: "{{ meta.name }}-db-management" kind: Deployment state: present @@ -52,16 +52,13 @@ wait: true - name: Check to make sure backup directory exists on PVC - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "stat {{ tower_backup_dir }}" register: stat_backup_dir -- debug: - msg: "{{stat_backup_dir}}" - - name: Error if backup dir is missing block: - name: Set error message diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/init_awx.yml index 47c74322..f3a215c1 100644 --- a/roles/restore/tasks/init_awx.yml +++ b/roles/restore/tasks/init_awx.yml @@ -1,7 +1,7 @@ --- - name: Get AWX object definition from pvc - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- @@ -22,10 +22,10 @@ apply: yes wait: yes template: "_definitions/awx_object.yml.j2" - wait: true -# TODO: The awx object and secrets need to be applied from the awx-operator, because that is where the service account is?. So we will need to either copy them over or pipe them into a template command +# TODO: The awx object and secrets need to be applied from the awx-operator, because that is where the service account is? +# So we will need to either copy them over or pipe them into a template command # TODO: Add logic to allow users to provide override values here, # or to specify spec values that were not in the backed up AWX object. diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index 44acf750..d1e4b980 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -67,7 +67,7 @@ -p {{ awx_postgres_port }} - name: Restore database dump to the new postgresql container - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ postgres_pod_name }}" command: | From 82efe053433b140ebcf02b1f18a219ed379401e2 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 8 Apr 2021 15:18:03 -0400 Subject: [PATCH 24/43] store secrets & definitions in a tempfile dir, fix postgres label --- roles/backup/tasks/awx-cro.yml | 4 ++-- roles/backup/tasks/cleanup.yml | 6 ------ roles/backup/tasks/postgres.yml | 2 +- roles/backup/tasks/secrets.yml | 24 ++++++++++++------------ roles/restore/tasks/apply_secret.yml | 4 ++-- roles/restore/tasks/cleanup.yml | 5 ----- roles/restore/tasks/init.yml | 8 ++++---- roles/restore/tasks/init_awx.yml | 6 +++--- roles/restore/tasks/postgres.yml | 14 ++++++++++---- 9 files changed, 34 insertions(+), 39 deletions(-) diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index 4db2f218..a143b75e 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -23,12 +23,12 @@ - name: Template AWX object definition template: src: awx_object.yml.j2 - dest: "_secrets/awx_object.yml" + dest: "{{ secrets_dir.path }}/awx_object.yml" mode: '0600' - name: Set AWX object template file as var set_fact: - awx_object_template: "{{ lookup('file', '_secrets/awx_object.yml') }}" + awx_object_template: "{{ lookup('file', '{{ secrets_dir.path }}/awx_object.yml') }}" - name: Write awx object to pvc k8s_exec: diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml index aa9c6b11..7e7e451b 100644 --- a/roles/backup/tasks/cleanup.yml +++ b/roles/backup/tasks/cleanup.yml @@ -1,11 +1,5 @@ --- -# After copying secret files to the PVC, delete the local tmp copies -- name: Clean up _secrets directory - file: - path: "_secrets" - state: absent - - name: Delete any existing management pod k8s: name: "{{ meta.name }}-db-management" diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index fa065780..25b5434f 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -32,7 +32,7 @@ kind: Pod namespace: '{{ meta.namespace }}' label_selectors: - - "app={{ tower_name }}-{{ deployment_type }}-postgres" + - "app.kubernetes.io/name={{ tower_name }}-postgres" register: postgres_pod until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" delay: 5 diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index bab87c52..047ab863 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -1,10 +1,10 @@ --- -- name: Make _secrets directory - file: - path: "_secrets" +- name: Make temp secrets directory + tempfile: + prefix: "secrets-" state: directory - mode: '0700' + register: secrets_dir - name: Get secret_key k8s_info: @@ -20,12 +20,12 @@ - name: Template secret_key definition template: src: secret_key_secret.yml.j2 - dest: "_secrets/secret_key_secret.yml" + dest: "{{ secrets_dir.path }}/secret_key_secret.yml" mode: '0700' - name: Set secret key template set_fact: - secret_key_template: "{{ lookup('file', '_secrets/secret_key_secret.yml') }}" + secret_key_template: "{{ lookup('file', '{{ secrets_dir.path }}/secret_key_secret.yml') }}" - name: Write secret_key to pvc k8s_exec: @@ -48,12 +48,12 @@ - name: Template admin_password definition template: src: admin_password_secret.yml.j2 - dest: "_secrets/admin_password_secret.yml" + dest: "{{ secrets_dir.path }}/admin_password_secret.yml" mode: '0700' - name: Set admin_password template set_fact: - admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yml') }}" + admin_password_template: "{{ lookup('file', '{{ secrets_dir.path }}/admin_password_secret.yml') }}" - name: Write secret_key to pvc k8s_exec: @@ -76,12 +76,12 @@ - name: Template broadcast_websocket definition template: src: broadcast_websocket_secret.yml.j2 - dest: "_secrets/broadcast_websocket_secret.yml" + dest: "{{ secrets_dir.path }}/broadcast_websocket_secret.yml" mode: '0700' - name: Set broadcast_websocket template set_fact: - broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml') }}" + broadcast_websocket_template: "{{ lookup('file', '{{ secrets_dir.path }}/broadcast_websocket_secret.yml') }}" - name: Write broadcast_websocket definition to pvc k8s_exec: @@ -109,12 +109,12 @@ - name: Template postgres configuration definition template: src: postgres_secret.yml.j2 - dest: "_secrets/postgres_secret.yml" + dest: "{{ secrets_dir.path }}/postgres_secret.yml" mode: '0700' - name: Set postgres configuration set_fact: - postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml') }}" + postgres_secret_template: "{{ lookup('file', '{{ secrets_dir.path }}/postgres_secret.yml') }}" - name: Write postgres configuration to pvc k8s_exec: diff --git a/roles/restore/tasks/apply_secret.yml b/roles/restore/tasks/apply_secret.yml index 095ac470..a71004cc 100644 --- a/roles/restore/tasks/apply_secret.yml +++ b/roles/restore/tasks/apply_secret.yml @@ -10,7 +10,7 @@ - name: Write temp secret definition template file copy: - dest: "_definitions/{{ item }}.yml.j2" + dest: "{{ definitions_dir.path }}/{{ item }}.yml.j2" content: | {{ awx_object.stdout }} mode: '0600' @@ -21,4 +21,4 @@ namespace: "{{ namespace | default('default') }}" apply: yes wait: yes - template: "_definitions/{{ item }}.yml.j2" + template: "{{ definitions_dir.path }}/{{ item }}.yml.j2" diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml index 6706916c..7e7e451b 100644 --- a/roles/restore/tasks/cleanup.yml +++ b/roles/restore/tasks/cleanup.yml @@ -1,10 +1,5 @@ --- -- name: Clean up _secrets directory - file: - path: "_definitions" - state: absent - - name: Delete any existing management pod k8s: name: "{{ meta.name }}-db-management" diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index 3a2280c8..5bde9fae 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -75,8 +75,8 @@ - tower_backup_dir != '' - stat_backup_dir.return_code != 0 -- name: Make _definitions directory - file: - path: "_definitions" +- name: Make temp definitions directory + tempfile: + prefix: "definitions-" state: directory - mode: '0700' + register: definitions_dir diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/init_awx.yml index f3a215c1..0ae71ad0 100644 --- a/roles/restore/tasks/init_awx.yml +++ b/roles/restore/tasks/init_awx.yml @@ -10,7 +10,7 @@ - name: Write temp AWX definition template file copy: - dest: "_definitions/awx_object.yml.j2" + dest: "{{ definitions_dir.path }}/awx_object.yml.j2" content: | {{ awx_object.stdout }} mode: '0600' @@ -18,10 +18,10 @@ - name: Deploy AWX k8s: state: "{{ state | default('present') }}" - namespace: "{{ namespace | default('default') }}" + namespace: "{{ meta.namespace | default('default') }}" apply: yes wait: yes - template: "_definitions/awx_object.yml.j2" + template: "{{ definitions_dir.path }}/awx_object.yml.j2" # TODO: The awx object and secrets need to be applied from the awx-operator, because that is where the service account is? diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index d1e4b980..75cdc62d 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -32,7 +32,7 @@ kind: Pod namespace: '{{ meta.namespace }}' label_selectors: - - "app={{ tower_name }}-{{ deployment_type }}-postgres" + - "app.kubernetes.io/name={{ tower_name }}-postgres" register: postgres_pod until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" delay: 5 @@ -59,17 +59,23 @@ replicas: 0 when: this_deployment['resources'] | length +- name: Get the postgres pod information + set_fact: + resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" + - name: Set pg_restore command set_fact: psql_restore: >- psql -U {{ awx_postgres_user }} - -d template1 + -h {{ resolvable_db_host }} + -U {{ awx_postgres_user }} + -d {{ awx_postgres_database }} -p {{ awx_postgres_port }} - name: Restore database dump to the new postgresql container k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ postgres_pod_name }}" + namespace: "{{ tower_backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" command: | bash -c """ set -e -o pipefail From 8ed0b1fe6194f2d59cd5edaf01e4b7ed8d7d0dae Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 9 Apr 2021 16:32:59 -0400 Subject: [PATCH 25/43] Template only what is needed from secrets and awx cro --- ansible/templates/awxrestore_crd.yml.j2 | 2 +- roles/backup/tasks/awx-cro.yml | 12 ++--- roles/backup/tasks/postgres.yml | 8 ++- roles/backup/tasks/secrets.yml | 69 +++--------------------- roles/backup/templates/awx_object.yml.j2 | 10 +--- roles/backup/templates/secrets.yml.j2 | 10 ++++ 6 files changed, 28 insertions(+), 83 deletions(-) create mode 100644 roles/backup/templates/secrets.yml.j2 diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 index e7e65923..e0ba2070 100644 --- a/ansible/templates/awxrestore_crd.yml.j2 +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -51,4 +51,4 @@ spec: description: Custom postgres_configuration secret name type: string oneOf: - - required: ["tower_name", "tower_backup_pvc"] + - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"] diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index a143b75e..b4d4056e 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -20,19 +20,13 @@ set_fact: awx_spec: "{{ _awx['spec'] }}" -- name: Template AWX object definition - template: - src: awx_object.yml.j2 - dest: "{{ secrets_dir.path }}/awx_object.yml" - mode: '0600' - -- name: Set AWX object template file as var +- name: Template secrets into yaml set_fact: - awx_object_template: "{{ lookup('file', '{{ secrets_dir.path }}/awx_object.yml') }}" + awx_definition_file: "{{ lookup('template', 'awx_object.yml.j2')}}" - name: Write awx object to pvc k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ awx_object_template }}' > {{ backup_dir }}/awx_object.yml" + bash -c "echo '{{ awx_definition_file }}' > {{ backup_dir }}/awx_object.yml" diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 25b5434f..07ed178e 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -69,13 +69,17 @@ namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db + bash -c "chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db" + +- name: Get the postgres pod information + set_fact: + resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" - name: Set pg_dump command set_fact: pgdump: >- pg_dump --clean --create - -h {{ awx_postgres_host }} + -h {{ resolvable_db_host }} -U {{ awx_postgres_user }} -d {{ awx_postgres_database }} -p {{ awx_postgres_port }} diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 047ab863..a250ddf7 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -1,8 +1,8 @@ --- - name: Make temp secrets directory - tempfile: - prefix: "secrets-" + file: + path: "/tmp/secrets" #-{{ lookup('password', '/dev/null chars=ascii_lowercase,digits length=8')}}" state: directory register: secrets_dir @@ -17,23 +17,6 @@ set_fact: secret_key: "{{ _secret_key['resources'][0]['data']['secret_key'] | b64decode }}" -- name: Template secret_key definition - template: - src: secret_key_secret.yml.j2 - dest: "{{ secrets_dir.path }}/secret_key_secret.yml" - mode: '0700' - -- name: Set secret key template - set_fact: - secret_key_template: "{{ lookup('file', '{{ secrets_dir.path }}/secret_key_secret.yml') }}" - -- name: Write secret_key to pvc - k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "echo '{{ secret_key_template }}' > {{ backup_dir }}/secret_key_secret.yml" - - name: Get admin_password k8s_info: kind: Secret @@ -45,23 +28,6 @@ set_fact: admin_password: "{{ _admin_password['resources'][0]['data']['password'] | b64decode }}" -- name: Template admin_password definition - template: - src: admin_password_secret.yml.j2 - dest: "{{ secrets_dir.path }}/admin_password_secret.yml" - mode: '0700' - -- name: Set admin_password template - set_fact: - admin_password_template: "{{ lookup('file', '{{ secrets_dir.path }}/admin_password_secret.yml') }}" - -- name: Write secret_key to pvc - k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "echo '{{ admin_password_template }}' > {{ backup_dir }}/admin_password_secret.yml" - - name: Get broadcast_websocket k8s_info: kind: Secret @@ -71,24 +37,7 @@ - name: Set broadcast_websocket key set_fact: - secret_key: "{{ _broadcast_websocket['resources'][0]['data']['secret'] | b64decode }}" - -- name: Template broadcast_websocket definition - template: - src: broadcast_websocket_secret.yml.j2 - dest: "{{ secrets_dir.path }}/broadcast_websocket_secret.yml" - mode: '0700' - -- name: Set broadcast_websocket template - set_fact: - broadcast_websocket_template: "{{ lookup('file', '{{ secrets_dir.path }}/broadcast_websocket_secret.yml') }}" - -- name: Write broadcast_websocket definition to pvc - k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "echo '{{ broadcast_websocket_template }}' > {{ backup_dir }}/broadcast_websocket_secret.yml" + broadcast_websocket: "{{ _broadcast_websocket['resources'][0]['data']['secret'] | b64decode }}" - name: Get postgres configuration k8s_info: @@ -106,19 +55,13 @@ database_host: "{{ _postgres_configuration['resources'][0]['data']['host'] | b64decode }}" database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode }}" -- name: Template postgres configuration definition - template: - src: postgres_secret.yml.j2 - dest: "{{ secrets_dir.path }}/postgres_secret.yml" - mode: '0700' - -- name: Set postgres configuration +- name: Template secrets into yaml set_fact: - postgres_secret_template: "{{ lookup('file', '{{ secrets_dir.path }}/postgres_secret.yml') }}" + secrets_file: "{{ lookup('template', 'secrets.yml.j2')}}" - name: Write postgres configuration to pvc k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ postgres_secret_template }}' > {{ backup_dir }}/postgres_secret.yml" + bash -c "echo '{{ secrets_file }}' > {{ backup_dir }}/secrets.yml" diff --git a/roles/backup/templates/awx_object.yml.j2 b/roles/backup/templates/awx_object.yml.j2 index fd4e9d92..4165a370 100644 --- a/roles/backup/templates/awx_object.yml.j2 +++ b/roles/backup/templates/awx_object.yml.j2 @@ -1,9 +1,3 @@ --- -apiVersion: '{{ awx_api_version }}' -kind: AWX -metadata: -{% raw %} - name: '{{ tower_name }}' - namespace: '{{ meta.namespace }}' -{% endraw %} -spec: {{ awx_spec }} +awx_api_version: {{ awx_api_version }} +awx_spec: {{ awx_spec }} diff --git a/roles/backup/templates/secrets.yml.j2 b/roles/backup/templates/secrets.yml.j2 new file mode 100644 index 00000000..94bacea2 --- /dev/null +++ b/roles/backup/templates/secrets.yml.j2 @@ -0,0 +1,10 @@ +--- +secret_key: {{ secret_key }} +admin_password: {{ admin_password }} +broadcast_websocket: {{ broadcast_websocket }} +database_password: {{ database_password }} +database_username: {{ database_username }} +database_name: {{ database_name }} +database_port: {{ database_port }} +database_host: {{ database_host }} +database_type: {{ database_type }} From fb612c24df737a4854fbf8c404d700fd46d14df7 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Mon, 12 Apr 2021 21:00:58 -0400 Subject: [PATCH 26/43] Only write values for spec section of awx object in backup --- roles/backup/tasks/awx-cro.yml | 8 +--- roles/backup/templates/awx_object.yml.j2 | 3 -- roles/restore/tasks/init_awx.yml | 13 ++---- roles/restore/tasks/init_secrets.yml | 22 +++++++--- roles/restore/tasks/secrets.yml | 21 ---------- roles/restore/templates/awx_object.yml.j2 | 7 ++++ roles/restore/templates/secrets.yml.j2 | 50 +++++++++++++++++++++++ 7 files changed, 79 insertions(+), 45 deletions(-) delete mode 100644 roles/backup/templates/awx_object.yml.j2 delete mode 100644 roles/restore/tasks/secrets.yml create mode 100644 roles/restore/templates/awx_object.yml.j2 create mode 100644 roles/restore/templates/secrets.yml.j2 diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index b4d4056e..3587ce70 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -12,21 +12,17 @@ set_fact: _awx: "{{ _awx_cro['resources'][0] }}" -- name: Set apiVersion - set_fact: - awx_api_version: "{{ _awx['apiVersion'] }}" - - name: Set user specified spec set_fact: awx_spec: "{{ _awx['spec'] }}" - name: Template secrets into yaml set_fact: - awx_definition_file: "{{ lookup('template', 'awx_object.yml.j2')}}" + awx_definition_file: "{{ awx_spec }}" - name: Write awx object to pvc k8s_exec: namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ awx_definition_file }}' > {{ backup_dir }}/awx_object.yml" + bash -c "echo '{{ awx_definition_file }}' > {{ backup_dir }}/awx_object" diff --git a/roles/backup/templates/awx_object.yml.j2 b/roles/backup/templates/awx_object.yml.j2 deleted file mode 100644 index 4165a370..00000000 --- a/roles/backup/templates/awx_object.yml.j2 +++ /dev/null @@ -1,3 +0,0 @@ ---- -awx_api_version: {{ awx_api_version }} -awx_spec: {{ awx_spec }} diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/init_awx.yml index 0ae71ad0..eb8c3bf9 100644 --- a/roles/restore/tasks/init_awx.yml +++ b/roles/restore/tasks/init_awx.yml @@ -5,15 +5,11 @@ namespace: "{{ tower_backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "cat '{{ tower_backup_dir }}/awx_object.yml'" + bash -c "cat '{{ tower_backup_dir }}/awx_object'" register: awx_object -- name: Write temp AWX definition template file - copy: - dest: "{{ definitions_dir.path }}/awx_object.yml.j2" - content: | - {{ awx_object.stdout }} - mode: '0600' +- set_fact: + awx_spec: "{{ awx_object.stdout }}" - name: Deploy AWX k8s: @@ -21,8 +17,7 @@ namespace: "{{ meta.namespace | default('default') }}" apply: yes wait: yes - template: "{{ definitions_dir.path }}/awx_object.yml.j2" - + template: awx_object.yml.j2 # TODO: The awx object and secrets need to be applied from the awx-operator, because that is where the service account is? # So we will need to either copy them over or pipe them into a template command diff --git a/roles/restore/tasks/init_secrets.yml b/roles/restore/tasks/init_secrets.yml index dab44259..636c4f68 100644 --- a/roles/restore/tasks/init_secrets.yml +++ b/roles/restore/tasks/init_secrets.yml @@ -1,9 +1,19 @@ --- - name: Get secret definition from pvc - include_tasks: apply_secrets.yml - with_items: - - secret_key_secret - - admin_password_secret - - broadcast_websocket_secret - - postgres_secret + k8s_exec: + namespace: "{{ tower_backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "cat '{{ tower_backup_dir }}/secrets.yml'" + register: secrets + +- include_vars: "{{ secrets.stdout | from_yaml }}" + +- name: Apply secret + k8s: + state: present + namespace: "{{ meta.namespace | default('default') }}" + apply: yes + wait: yes + template: "secrets.yml.j2" diff --git a/roles/restore/tasks/secrets.yml b/roles/restore/tasks/secrets.yml deleted file mode 100644 index 705cf133..00000000 --- a/roles/restore/tasks/secrets.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- - -- name: Create secret_key secret - k8s: - definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'secret_key_secret.yml'])) }}" - -- name: Create admin_password secret - k8s: - definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'admin_password_secret.yml'])) }}" - -- name: Create broadcast_websocket secret - k8s: - definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'broadcast_websocket_secret.yml'])) }}" - -- name: Create postgres configuration secret - k8s: - definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'postgres_secret.yml'])) }}" - -- name: Create secret_key secret - k8s: - definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'secret_key_secret.yml'])) }}" diff --git a/roles/restore/templates/awx_object.yml.j2 b/roles/restore/templates/awx_object.yml.j2 new file mode 100644 index 00000000..e9638b48 --- /dev/null +++ b/roles/restore/templates/awx_object.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: '{{ awx_api_version }}' +kind: AWX +metadata: + name: '{{ tower_name }}' + namespace: '{{ meta.namespace }}' +spec: {{ awx_spec }} diff --git a/roles/restore/templates/secrets.yml.j2 b/roles/restore/templates/secrets.yml.j2 new file mode 100644 index 00000000..a4fee4fe --- /dev/null +++ b/roles/restore/templates/secrets.yml.j2 @@ -0,0 +1,50 @@ +# Postgres Secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: '{{ tower_name }}-postgres-configuration' + namespace: '{{ meta.namespace }}' +stringData: + password: '{{ database_password }}' + username: '{{ database_username }}' + database: '{{ database_name }}' + port: '{{ database_port }}' + host: '{{ database_host }}' + type: '{{ database_type }}' + +# Secret Key Secret +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ tower_name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + secret_key: '{{ secret_key }}' + +# Admin Password Secret +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ tower_name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + password: '{{ admin_password }}' + +# Broadcast Websocket Secret +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ tower_name }}-broadcast-websocket' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + secret: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}' From 5b32c412770c602f9320873f3b2921377b3ad6df Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 13 Apr 2021 14:47:38 -0400 Subject: [PATCH 27/43] Fix retry for checking postgres pod & fix secrets template - fixed a lot of typos & updated the README.md files --- ansible/templates/awxrestore_crd.yml.j2 | 6 +-- ansible/templates/role.yml.j2 | 1 + deploy/awx-operator.yaml | 11 +++--- deploy/crds/awxrestore_v1beta1_crd.yaml | 8 ++-- roles/backup/README.md | 24 +++++++----- roles/backup/meta/main.yml | 1 + roles/backup/tasks/init.yml | 2 +- roles/backup/tasks/main.yml | 2 +- roles/backup/tasks/postgres.yml | 6 ++- roles/backup/tasks/secrets.yml | 10 +---- .../templates/admin_password_secret.yml.j2 | 10 ----- .../broadcast_websocket_secret.yml.j2 | 10 ----- roles/backup/templates/postgres_secret.yml.j2 | 16 -------- .../backup/templates/secret_key_secret.yml.j2 | 10 ----- roles/restore/README.md | 12 +++--- roles/restore/meta/main.yml | 1 + roles/restore/tasks/apply_secret.yml | 24 ------------ .../tasks/{init_awx.yml => deploy_awx.yml} | 11 +++--- roles/restore/tasks/init_secrets.yml | 19 ---------- roles/restore/tasks/main.yml | 6 +-- roles/restore/tasks/postgres.yml | 6 ++- roles/restore/tasks/secrets.yml | 37 +++++++++++++++++++ roles/restore/templates/awx_object.yml.j2 | 2 +- roles/restore/templates/secrets.yml.j2 | 6 --- 24 files changed, 97 insertions(+), 144 deletions(-) delete mode 100644 roles/backup/templates/admin_password_secret.yml.j2 delete mode 100644 roles/backup/templates/broadcast_websocket_secret.yml.j2 delete mode 100644 roles/backup/templates/postgres_secret.yml.j2 delete mode 100644 roles/backup/templates/secret_key_secret.yml.j2 delete mode 100644 roles/restore/tasks/apply_secret.yml rename roles/restore/tasks/{init_awx.yml => deploy_awx.yml} (74%) delete mode 100644 roles/restore/tasks/init_secrets.yml create mode 100644 roles/restore/tasks/secrets.yml diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 index e0ba2070..07f59d48 100644 --- a/ansible/templates/awxrestore_crd.yml.j2 +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -21,7 +21,7 @@ spec: openAPIV3Schema: type: object x-kubernetes-preserve-unknown-fields: true - description: Schema validation for the AWXBackup CRD + description: Schema validation for the AWXRestore CRD properties: spec: type: object @@ -30,10 +30,10 @@ spec: description: Name of the deployment to be restored to type: string tower_backup_pvc: - description: Name of the PVC to be used for storing the backup + description: Name of the PVC to be restored from type: string tower_backup_pvc_namespace: - description: Namespace PVC is in + description: Namespace the PVC is in type: string tower_backup_dir: description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) diff --git a/ansible/templates/role.yml.j2 b/ansible/templates/role.yml.j2 index 80405470..886bd5f0 100644 --- a/ansible/templates/role.yml.j2 +++ b/ansible/templates/role.yml.j2 @@ -80,5 +80,6 @@ rules: resources: - '*' - awxbackups + - awxrestores verbs: - '*' diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index d5119824..23325e41 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -457,7 +457,7 @@ spec: openAPIV3Schema: type: object x-kubernetes-preserve-unknown-fields: true - description: Schema validation for the AWXBackup CRD + description: Schema validation for the AWXRestore CRD properties: spec: type: object @@ -466,10 +466,10 @@ spec: description: Name of the deployment to be restored to type: string tower_backup_pvc: - description: Name of the PVC to be used for storing the backup + description: Name of the PVC to be restored from type: string tower_backup_pvc_namespace: - description: Namespace PVC is in + description: Namespace the PVC is in type: string tower_backup_dir: description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) @@ -487,7 +487,7 @@ spec: description: Custom postgres_configuration secret name type: string oneOf: - - required: ["tower_name", "tower_backup_pvc"] + - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -570,6 +570,7 @@ rules: resources: - '*' - awxbackups + - awxrestores verbs: - '*' @@ -612,7 +613,7 @@ spec: serviceAccountName: awx-operator containers: - name: awx-operator - image: "quay.io/chadams/awx-operator:new-crd" + image: "quay.io/ansible/awx-operator:0.8.0" imagePullPolicy: "Always" volumeMounts: - mountPath: /tmp/ansible-operator/runner diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml index e7e65923..07f59d48 100644 --- a/deploy/crds/awxrestore_v1beta1_crd.yaml +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -21,7 +21,7 @@ spec: openAPIV3Schema: type: object x-kubernetes-preserve-unknown-fields: true - description: Schema validation for the AWXBackup CRD + description: Schema validation for the AWXRestore CRD properties: spec: type: object @@ -30,10 +30,10 @@ spec: description: Name of the deployment to be restored to type: string tower_backup_pvc: - description: Name of the PVC to be used for storing the backup + description: Name of the PVC to be restored from type: string tower_backup_pvc_namespace: - description: Namespace PVC is in + description: Namespace the PVC is in type: string tower_backup_dir: description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) @@ -51,4 +51,4 @@ spec: description: Custom postgres_configuration secret name type: string oneOf: - - required: ["tower_name", "tower_backup_pvc"] + - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"] diff --git a/roles/backup/README.md b/roles/backup/README.md index 3d93035c..46215b85 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -1,12 +1,11 @@ -Role Name +Backup Role ========= -The purpose of this role is to create a backup of your AWX deployment. This includes: +The purpose of this role is to create a backup of your AWX deployment which includes: + - custom deployment specific values in the spec section of the AWX custom resource object - backup of the postgresql database - - secret_key - - custom user config files - - manual projects - + - secret_key, admin_password, and broadcast_websocket secrets + - database configuration Requirements ------------ @@ -32,13 +31,12 @@ spec: tower_name: mytower ``` -Note that the `tower_name` above is the name of the AWX deployment you intend to backup from. - +Note that the `tower_name` above is the name of the AWX deployment you intend to backup from. The namespace above is the one containing the AWX deployment that will be backed up. Finally, use `kubectl` to create the backup object in your cluster: ```bash -#> kubectl apply -f backup-awx.yml +$ kubectl apply -f backup-awx.yml ``` The resulting pvc will contain a backup tar that can be used to restore to a new deployment. Future backups will also be stored in separate tars on the same pvc. @@ -62,6 +60,14 @@ tower_backup_storage_class: 'standard' tower_backup_size: '20Gi' ``` +By default, the backup pvc will be created in the `default` namespace. If you want your backup to be stored +in a specific namespace, you can do so by specifying `tower_backup_pvc_namespace`. Keep in mind that you will +need to provide the same namespace when restoring. + +``` +tower_backup_pvc_namespace: 'custom-namespace' +``` + If a custom postgres configuration secret was used when deploying AWX, it must be set: ``` diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml index c48a9cf2..3d3cd361 100644 --- a/roles/backup/meta/main.yml +++ b/roles/backup/meta/main.yml @@ -18,6 +18,7 @@ galaxy_info: galaxy_tags: - tower + - controller - awx - ansible - backup diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 119c68e1..69699796 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -40,7 +40,7 @@ set_fact: _default_backup_pvc: "{{ meta.name }}-backup-claim" -# by default, it will re-use the old pvc if already created (unless pvc is provided) +# by default, it will re-use the old pvc if already created (unless a pvc is provided) - name: Set PVC to use for backup set_fact: backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index e682e860..84f3ead3 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -34,4 +34,4 @@ - name: Update status variables include_tasks: update_status.yml -# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object. See ticket +# TODO: backup tower settings or make sure that users only specify settings/config changes via AWX object. See ticket diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 07ed178e..e9ca54ce 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -34,7 +34,9 @@ label_selectors: - "app.kubernetes.io/name={{ tower_name }}-postgres" register: postgres_pod - until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" delay: 5 retries: 60 @@ -71,7 +73,7 @@ command: >- bash -c "chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db" -- name: Get the postgres pod information +- name: Set full resolvable host name for postgres pod set_fact: resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index a250ddf7..6ef987b4 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -1,11 +1,5 @@ --- -- name: Make temp secrets directory - file: - path: "/tmp/secrets" #-{{ lookup('password', '/dev/null chars=ascii_lowercase,digits length=8')}}" - state: directory - register: secrets_dir - - name: Get secret_key k8s_info: kind: Secret @@ -42,7 +36,7 @@ - name: Get postgres configuration k8s_info: kind: Secret - namespace: '{{ tower_backup_pvc_namespace }}' + namespace: '{{ meta.namespace }}' name: '{{ tower_postgres_configuration_secret }}' register: _postgres_configuration @@ -57,7 +51,7 @@ - name: Template secrets into yaml set_fact: - secrets_file: "{{ lookup('template', 'secrets.yml.j2')}}" + secrets_file: "{{ lookup('template', 'secrets.yml.j2') }}" - name: Write postgres configuration to pvc k8s_exec: diff --git a/roles/backup/templates/admin_password_secret.yml.j2 b/roles/backup/templates/admin_password_secret.yml.j2 deleted file mode 100644 index 8adc8177..00000000 --- a/roles/backup/templates/admin_password_secret.yml.j2 +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: -{% raw %} - name: '{{ tower_name }}' - namespace: '{{ meta.namespace }}' -{% endraw %} -stringData: - password: '{{ admin_password }}' diff --git a/roles/backup/templates/broadcast_websocket_secret.yml.j2 b/roles/backup/templates/broadcast_websocket_secret.yml.j2 deleted file mode 100644 index e590ab06..00000000 --- a/roles/backup/templates/broadcast_websocket_secret.yml.j2 +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: -{% raw %} - name: '{{ tower_name }}-broadcast-websocket' - namespace: '{{ meta.namespace }}' -{% endraw %} -stringData: - secret: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}' diff --git a/roles/backup/templates/postgres_secret.yml.j2 b/roles/backup/templates/postgres_secret.yml.j2 deleted file mode 100644 index f5fa75d1..00000000 --- a/roles/backup/templates/postgres_secret.yml.j2 +++ /dev/null @@ -1,16 +0,0 @@ -# Postgres Secret. ---- -apiVersion: v1 -kind: Secret -metadata: -{% raw %} - name: '{{ tower_name }}-postgres-configuration' - namespace: '{{ meta.namespace }}' -{% endraw %} -stringData: - password: '{{ database_password }}' - username: '{{ database_username }}' - database: '{{ database_name }}' - port: '{{ database_port }}' - host: '{{ database_host }}' - type: '{{ database_type }}' diff --git a/roles/backup/templates/secret_key_secret.yml.j2 b/roles/backup/templates/secret_key_secret.yml.j2 deleted file mode 100644 index ae60b37f..00000000 --- a/roles/backup/templates/secret_key_secret.yml.j2 +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: -{% raw %} - name: '{{ tower_name }}' - namespace: '{{ meta.namespace }}' -{% endraw %} -stringData: - secret_key: '{{ secret_key }}' diff --git a/roles/restore/README.md b/roles/restore/README.md index ae262c11..0d9eb1fb 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -1,10 +1,12 @@ -Role Name +Restore Role ========= -The purpose of this role is to restore your AWX deployment from an existing PVC backup. The backup should include: +The purpose of this role is to restore your AWX deployment from an existing PVC backup. The backup includes: + - custom deployment specific values in the spec section of the AWX custom resource object - backup of the postgresql database - - secrets, included the secret_key. - - AWX custom resource object with deployment specific settings + - secret_key, admin_password, and broadcast_websocket secrets + - database configuration + Requirements @@ -44,7 +46,7 @@ kubectl create ns my-namespace Finally, use `kubectl` to create the restore object in your cluster: ```bash -#> kubectl apply -f restore-awx.yml +$ kubectl apply -f restore-awx.yml ``` This will create a new deployment and restore your backup to it. diff --git a/roles/restore/meta/main.yml b/roles/restore/meta/main.yml index f9e95b3b..720b9bcb 100644 --- a/roles/restore/meta/main.yml +++ b/roles/restore/meta/main.yml @@ -18,6 +18,7 @@ galaxy_info: galaxy_tags: - tower + - controller - awx - ansible - restore diff --git a/roles/restore/tasks/apply_secret.yml b/roles/restore/tasks/apply_secret.yml deleted file mode 100644 index a71004cc..00000000 --- a/roles/restore/tasks/apply_secret.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -- name: Get secret definition from pvc - k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "cat '{{ tower_backup_dir }}/{{ item }}.yml'" - register: awx_object - -- name: Write temp secret definition template file - copy: - dest: "{{ definitions_dir.path }}/{{ item }}.yml.j2" - content: | - {{ awx_object.stdout }} - mode: '0600' - -- name: Apply secret - k8s: - state: "{{ state | default('present') }}" - namespace: "{{ namespace | default('default') }}" - apply: yes - wait: yes - template: "{{ definitions_dir.path }}/{{ item }}.yml.j2" diff --git a/roles/restore/tasks/init_awx.yml b/roles/restore/tasks/deploy_awx.yml similarity index 74% rename from roles/restore/tasks/init_awx.yml rename to roles/restore/tasks/deploy_awx.yml index eb8c3bf9..4332c5e3 100644 --- a/roles/restore/tasks/init_awx.yml +++ b/roles/restore/tasks/deploy_awx.yml @@ -8,7 +8,8 @@ bash -c "cat '{{ tower_backup_dir }}/awx_object'" register: awx_object -- set_fact: +- name: Set AWX spec variable from backup + set_fact: awx_spec: "{{ awx_object.stdout }}" - name: Deploy AWX @@ -16,11 +17,11 @@ state: "{{ state | default('present') }}" namespace: "{{ meta.namespace | default('default') }}" apply: yes - wait: yes template: awx_object.yml.j2 - -# TODO: The awx object and secrets need to be applied from the awx-operator, because that is where the service account is? -# So we will need to either copy them over or pipe them into a template command + wait: true + wait_condition: + type: "Running" + status: "True" # TODO: Add logic to allow users to provide override values here, # or to specify spec values that were not in the backed up AWX object. diff --git a/roles/restore/tasks/init_secrets.yml b/roles/restore/tasks/init_secrets.yml deleted file mode 100644 index 636c4f68..00000000 --- a/roles/restore/tasks/init_secrets.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -- name: Get secret definition from pvc - k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" - pod: "{{ meta.name }}-db-management" - command: >- - bash -c "cat '{{ tower_backup_dir }}/secrets.yml'" - register: secrets - -- include_vars: "{{ secrets.stdout | from_yaml }}" - -- name: Apply secret - k8s: - state: present - namespace: "{{ meta.namespace | default('default') }}" - apply: yes - wait: yes - template: "secrets.yml.j2" diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index 63d1138d..48461ba3 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -16,12 +16,12 @@ - block: - include_tasks: init.yml - - include_tasks: init_awx.yml + - include_tasks: secrets.yml + + - include_tasks: deploy_awx.yml - include_tasks: postgres.yml - - include_tasks: secrets.yml - - name: Set flag signifying this restore was successful set_fact: tower_restore_complete: True diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index 75cdc62d..13e38816 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -34,7 +34,9 @@ label_selectors: - "app.kubernetes.io/name={{ tower_name }}-postgres" register: postgres_pod - until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" delay: 5 retries: 60 @@ -59,7 +61,7 @@ replicas: 0 when: this_deployment['resources'] | length -- name: Get the postgres pod information +- name: Set full resolvable host name for postgres pod set_fact: resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" diff --git a/roles/restore/tasks/secrets.yml b/roles/restore/tasks/secrets.yml new file mode 100644 index 00000000..1a9cd864 --- /dev/null +++ b/roles/restore/tasks/secrets.yml @@ -0,0 +1,37 @@ +--- + +- name: Get secret definition from pvc + k8s_exec: + namespace: "{{ tower_backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "cat '{{ tower_backup_dir }}/secrets.yml'" + register: secrets + +- name: Create temp vars file + tempfile: + prefix: secret_vars- + register: secret_vars + +- name: Write vars to file locally + copy: + dest: "{{ secret_vars.path }}" + content: "{{ secrets.stdout }}" + mode: 0640 + +- name: Include secret vars from backup + include_vars: "{{ secret_vars.path }}" + +- name: Set new database host based on supplied tower_name + set_fact: + database_host: "{{ tower_name }}-postgres" + when: + - database_type == 'managed' + +- name: Apply secret + k8s: + state: present + namespace: "{{ meta.namespace | default('default') }}" + apply: yes + wait: yes + template: "secrets.yml.j2" diff --git a/roles/restore/templates/awx_object.yml.j2 b/roles/restore/templates/awx_object.yml.j2 index e9638b48..b13e3a51 100644 --- a/roles/restore/templates/awx_object.yml.j2 +++ b/roles/restore/templates/awx_object.yml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: '{{ awx_api_version }}' +apiVersion: '{{ api_version }}' kind: AWX metadata: name: '{{ tower_name }}' diff --git a/roles/restore/templates/secrets.yml.j2 b/roles/restore/templates/secrets.yml.j2 index a4fee4fe..019af5fa 100644 --- a/roles/restore/templates/secrets.yml.j2 +++ b/roles/restore/templates/secrets.yml.j2 @@ -18,10 +18,8 @@ stringData: apiVersion: v1 kind: Secret metadata: -{% raw %} name: '{{ tower_name }}' namespace: '{{ meta.namespace }}' -{% endraw %} stringData: secret_key: '{{ secret_key }}' @@ -30,10 +28,8 @@ stringData: apiVersion: v1 kind: Secret metadata: -{% raw %} name: '{{ tower_name }}' namespace: '{{ meta.namespace }}' -{% endraw %} stringData: password: '{{ admin_password }}' @@ -42,9 +38,7 @@ stringData: apiVersion: v1 kind: Secret metadata: -{% raw %} name: '{{ tower_name }}-broadcast-websocket' namespace: '{{ meta.namespace }}' -{% endraw %} stringData: secret: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}' From 8f760e2842d98f34b1a3a0e9d12f3640db8c36b2 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 16 Apr 2021 09:32:34 -0400 Subject: [PATCH 28/43] Allow users to change pg name, pw & db name for a managed postgres - set default value for postgres-configuration type as unmanaged if secret is created --- README.md | 3 +++ roles/backup/tasks/secrets.yml | 2 +- roles/installer/tasks/migrate_data.yml | 4 +++- roles/installer/templates/tower_postgres.yaml.j2 | 6 +++--- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0ab9d6de..65edb8d4 100644 --- a/README.md +++ b/README.md @@ -229,9 +229,12 @@ stringData: username: password: sslmode: prefer + type: unmanaged type: Opaque ``` +> It is possible to set a specific username, password, or database, but still have the database managed by the operator. In this case, when creating the postgres-configuration secret, the `type: managed` field should be added. + **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`. #### Migrating data from an old AWX instance diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 6ef987b4..66bc77ea 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -47,7 +47,7 @@ database_name: "{{ _postgres_configuration['resources'][0]['data']['database'] | b64decode }}" database_port: "{{ _postgres_configuration['resources'][0]['data']['port'] | b64decode }}" database_host: "{{ _postgres_configuration['resources'][0]['data']['host'] | b64decode }}" - database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode }}" + database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode | default('unmanaged')}}" - name: Template secrets into yaml set_fact: diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index 28604306..ca1bb7ae 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -16,7 +16,9 @@ field_selectors: - status.phase=Running register: postgres_pod - until: postgres_pod['resources'] | length + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" delay: 5 retries: 60 diff --git a/roles/installer/templates/tower_postgres.yaml.j2 b/roles/installer/templates/tower_postgres.yaml.j2 index a98086a8..98a8b4bd 100644 --- a/roles/installer/templates/tower_postgres.yaml.j2 +++ b/roles/installer/templates/tower_postgres.yaml.j2 @@ -53,17 +53,17 @@ spec: - name: POSTGRES_DB valueFrom: secretKeyRef: - name: '{{ meta.name }}-postgres-configuration' + name: '{{ tower_postgres_configuration_secret }}' key: database - name: POSTGRES_USER valueFrom: secretKeyRef: - name: '{{ meta.name }}-postgres-configuration' + name: '{{ tower_postgres_configuration_secret }}' key: username - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: '{{ meta.name }}-postgres-configuration' + name: '{{ tower_postgres_configuration_secret }}' key: password - name: PGDATA value: '{{ tower_postgres_data_path }}' From 90f4d716062cc570a640d538dbae7e57d079d8b5 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 16 Apr 2021 09:54:19 -0400 Subject: [PATCH 29/43] Make pg port configurable for managed deployments --- README.md | 2 +- roles/installer/tasks/database_configuration.yml | 7 +++++++ roles/installer/templates/tower_postgres.yaml.j2 | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 65edb8d4..aede3d3e 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ stringData: type: Opaque ``` -> It is possible to set a specific username, password, or database, but still have the database managed by the operator. In this case, when creating the postgres-configuration secret, the `type: managed` field should be added. +> It is possible to set a specific username, password, port, or database, but still have the database managed by the operator. In this case, when creating the postgres-configuration secret, the `type: managed` field should be added. **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`. diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index 1f52509d..ecebe506 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -110,6 +110,13 @@ awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}" +- name: Create Database if no database is specified + k8s: + apply: true + definition: "{{ lookup('template', 'tower_postgres.yaml.j2') }}" + when: + - pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' + - name: Set apiVersion and kind variables set_fact: api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' diff --git a/roles/installer/templates/tower_postgres.yaml.j2 b/roles/installer/templates/tower_postgres.yaml.j2 index 98a8b4bd..c73bd88f 100644 --- a/roles/installer/templates/tower_postgres.yaml.j2 +++ b/roles/installer/templates/tower_postgres.yaml.j2 @@ -72,7 +72,7 @@ spec: - name: POSTGRES_HOST_AUTH_METHOD value: '{{ postgres_host_auth_method }}' ports: - - containerPort: 5432 + - containerPort: '{{ awx_postgres_port }}' name: postgres volumeMounts: - name: postgres From 38a6a02f859f1bcd352bb310013ce9dbfb8a919c Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 16 Apr 2021 11:07:52 -0400 Subject: [PATCH 30/43] Add secret names as statuses on the AWX object - set migrate data status even if custom name for old postgres config is not used - Allow users to change pg name, pw & db name for a managed postgres - set default value for postgres-configuration type as unmanaged if secret is created - Make pg port configurable for managed deployments --- ansible/templates/crd.yml.j2 | 11 +++++++- deploy/awx-operator.yaml | 11 +++++++- deploy/crds/awx_v1beta1_crd.yaml | 11 +++++++- roles/backup/tasks/main.yml | 2 +- roles/backup/tasks/secrets.yml | 22 +++++++++++---- .../tasks/database_configuration.yml | 7 ----- roles/installer/tasks/migrate_data.yml | 6 ++++- roles/installer/tasks/update_status.yml | 27 +++++++++++++++++++ .../templates/tower_postgres.yaml.j2 | 8 +++--- roles/restore/tasks/main.yml | 4 +-- roles/restore/templates/secrets.yml.j2 | 6 ++--- 11 files changed, 89 insertions(+), 26 deletions(-) diff --git a/ansible/templates/crd.yml.j2 b/ansible/templates/crd.yml.j2 index ad8ca846..c5956c1f 100644 --- a/ansible/templates/crd.yml.j2 +++ b/ansible/templates/crd.yml.j2 @@ -346,7 +346,16 @@ spec: description: Admin user of the deployed instance type: string towerAdminPasswordSecret: - description: Admin password of the deployed instance + description: Admin password secret name of the deployed instance + type: string + towerPostgresConfigurationSecret: + description: Postgres Configuration secret name of the deployed instance + type: string + towerBroadcastWebsocketSecret: + description: Broadcast websocket secret name of the deployed instance + type: string + towerSecretKeySecret: + description: Secret key secret name of the deployed instance type: string towerMigratedFromSecret: description: The secret used for migrating an old Tower. diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 23325e41..3513684f 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -348,7 +348,16 @@ spec: description: Admin user of the deployed instance type: string towerAdminPasswordSecret: - description: Admin password of the deployed instance + description: Admin password secret name of the deployed instance + type: string + towerPostgresConfigurationSecret: + description: Postgres Configuration secret name of the deployed instance + type: string + towerBroadcastWebsocketSecret: + description: Broadcast websocket secret name of the deployed instance + type: string + towerSecretKeySecret: + description: Secret key secret name of the deployed instance type: string towerMigratedFromSecret: description: The secret used for migrating an old Tower. diff --git a/deploy/crds/awx_v1beta1_crd.yaml b/deploy/crds/awx_v1beta1_crd.yaml index ad8ca846..c5956c1f 100644 --- a/deploy/crds/awx_v1beta1_crd.yaml +++ b/deploy/crds/awx_v1beta1_crd.yaml @@ -346,7 +346,16 @@ spec: description: Admin user of the deployed instance type: string towerAdminPasswordSecret: - description: Admin password of the deployed instance + description: Admin password secret name of the deployed instance + type: string + towerPostgresConfigurationSecret: + description: Postgres Configuration secret name of the deployed instance + type: string + towerBroadcastWebsocketSecret: + description: Broadcast websocket secret name of the deployed instance + type: string + towerSecretKeySecret: + description: Secret key secret name of the deployed instance type: string towerMigratedFromSecret: description: The secret used for migrating an old Tower. diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 84f3ead3..0966577f 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -5,7 +5,7 @@ api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' -- name: Look up details for this deployment +- name: Look up details for this backup object k8s_info: api_version: "{{ api_version }}" kind: "{{ kind }}" diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 66bc77ea..232c2ab8 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -1,10 +1,22 @@ --- +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + +- name: Look up details for this deployment + k8s_info: + api_version: "{{ api_version }}" + kind: "AWX" # Find a way to dynamically get this + name: "{{ tower_name }}" + namespace: "{{ meta.namespace }}" + register: this_awx + - name: Get secret_key k8s_info: kind: Secret namespace: '{{ meta.namespace }}' - name: '{{ tower_secret_key_secret }}' + name: "{{ this_awx['resources'][0]['status']['towerSecretKeySecret'] }}" register: _secret_key - name: Set secret key @@ -15,7 +27,7 @@ k8s_info: kind: Secret namespace: '{{ meta.namespace }}' - name: '{{ tower_admin_password_secret }}' + name: "{{ this_awx['resources'][0]['status']['towerAdminPasswordSecret'] }}" register: _admin_password - name: Set admin_password @@ -26,7 +38,7 @@ k8s_info: kind: Secret namespace: '{{ meta.namespace }}' - name: '{{ tower_broadcast_websocket_secret }}' + name: "{{ this_awx['resources'][0]['status']['towerBroadcastWebsocketSecret'] }}" register: _broadcast_websocket - name: Set broadcast_websocket key @@ -37,7 +49,7 @@ k8s_info: kind: Secret namespace: '{{ meta.namespace }}' - name: '{{ tower_postgres_configuration_secret }}' + name: "{{ this_awx['resources'][0]['status']['towerPostgresConfigurationSecret'] }}" register: _postgres_configuration - name: Set postgres configuration @@ -47,7 +59,7 @@ database_name: "{{ _postgres_configuration['resources'][0]['data']['database'] | b64decode }}" database_port: "{{ _postgres_configuration['resources'][0]['data']['port'] | b64decode }}" database_host: "{{ _postgres_configuration['resources'][0]['data']['host'] | b64decode }}" - database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode | default('unmanaged')}}" + database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" - name: Template secrets into yaml set_fact: diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index ecebe506..9c24b1fb 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -100,7 +100,6 @@ definition: "{{ lookup('template', 'tower_postgres.yaml.j2') }}" when: pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' - - name: Store Database Configuration set_fact: awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" @@ -110,12 +109,6 @@ awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}" -- name: Create Database if no database is specified - k8s: - apply: true - definition: "{{ lookup('template', 'tower_postgres.yaml.j2') }}" - when: - - pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' - name: Set apiVersion and kind variables set_fact: diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index ca1bb7ae..e6bbab80 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -1,5 +1,9 @@ --- +- name: Set actual old postgres configuration secret name + set_fact: + old_postgres_configuration_name: "{{ old_pg_config['resources'][0]['metadata']['name'] }}" + - name: Store Database Configuration set_fact: awx_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}" @@ -60,4 +64,4 @@ - name: Set flag signifying that this instance has been migrated set_fact: - tower_migrated_from_secret: "{{ tower_old_postgres_configuration_secret }}" + tower_migrated_from_secret: "{{ old_postgres_configuration_name }}" diff --git a/roles/installer/tasks/update_status.yml b/roles/installer/tasks/update_status.yml index 0b123d9a..c3194992 100644 --- a/roles/installer/tasks/update_status.yml +++ b/roles/installer/tasks/update_status.yml @@ -17,6 +17,33 @@ status: towerAdminUser: "{{ tower_admin_user }}" +- name: Update postgres configuration status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerPostgresConfigurationSecret: "{{ pg_config['resources'][0]['metadata']['name'] }}" + +- name: Update broadcast websocket status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerBroadcastWebsocketSecret: "{{ broadcast_websocket_secret['resources'][0]['metadata']['name'] }}" + +- name: Update secret key status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerSecretKeySecret: "{{ secret_key_secret_name }}" + - name: Retrieve instance version k8s_exec: namespace: "{{ meta.namespace }}" diff --git a/roles/installer/templates/tower_postgres.yaml.j2 b/roles/installer/templates/tower_postgres.yaml.j2 index c73bd88f..694a6940 100644 --- a/roles/installer/templates/tower_postgres.yaml.j2 +++ b/roles/installer/templates/tower_postgres.yaml.j2 @@ -53,17 +53,17 @@ spec: - name: POSTGRES_DB valueFrom: secretKeyRef: - name: '{{ tower_postgres_configuration_secret }}' + name: '{{ postgres_configuration_secret }}' key: database - name: POSTGRES_USER valueFrom: secretKeyRef: - name: '{{ tower_postgres_configuration_secret }}' + name: '{{ postgres_configuration_secret }}' key: username - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: '{{ tower_postgres_configuration_secret }}' + name: '{{ postgres_configuration_secret }}' key: password - name: PGDATA value: '{{ tower_postgres_data_path }}' @@ -72,7 +72,7 @@ spec: - name: POSTGRES_HOST_AUTH_METHOD value: '{{ postgres_host_auth_method }}' ports: - - containerPort: '{{ awx_postgres_port }}' + - containerPort: {{ awx_postgres_port | default('5432')}} name: postgres volumeMounts: - name: postgres diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index 48461ba3..7fda2e1c 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -5,7 +5,7 @@ api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' -- name: Look up details for this deployment +- name: Look up details for this restore object k8s_info: api_version: "{{ api_version }}" kind: "{{ kind }}" @@ -34,4 +34,4 @@ - name: Update status variables include_tasks: update_status.yml -# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object. See ticket +# TODO: backup tower settings or make sure that users only specify settings/config changes via AWX object. See ticket diff --git a/roles/restore/templates/secrets.yml.j2 b/roles/restore/templates/secrets.yml.j2 index 019af5fa..39bca5c8 100644 --- a/roles/restore/templates/secrets.yml.j2 +++ b/roles/restore/templates/secrets.yml.j2 @@ -18,7 +18,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: '{{ tower_name }}' + name: '{{ tower_name }}-secret-key' namespace: '{{ meta.namespace }}' stringData: secret_key: '{{ secret_key }}' @@ -28,7 +28,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: '{{ tower_name }}' + name: '{{ tower_name }}-admin-password' namespace: '{{ meta.namespace }}' stringData: password: '{{ admin_password }}' @@ -41,4 +41,4 @@ metadata: name: '{{ tower_name }}-broadcast-websocket' namespace: '{{ meta.namespace }}' stringData: - secret: '{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}' + secret: '{{ broadcast_websocket }}' From ff9248e971ad836169ac54b61046013d4bec5973 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 22 Apr 2021 12:37:13 -0400 Subject: [PATCH 31/43] create pvc in namespace of old awx by default, update docs, fix bug with secret statuses --- ansible/templates/awxrestore_crd.yml.j2 | 2 +- deploy/awx-operator.yaml | 2 +- deploy/crds/awxrestore_v1beta1_crd.yaml | 2 +- roles/backup/README.md | 14 +++++--------- roles/backup/defaults/main.yml | 8 +------- roles/backup/tasks/init.yml | 13 +++++++++++++ roles/backup/tasks/postgres.yml | 19 ++++++------------- roles/backup/tasks/secrets.yml | 12 ------------ roles/restore/README.md | 11 +++++++++-- roles/restore/defaults/main.yml | 9 ++++++--- roles/restore/tasks/deploy_awx.yml | 2 +- roles/restore/tasks/init.yml | 6 ------ roles/restore/tasks/secrets.yml | 2 +- 13 files changed, 45 insertions(+), 57 deletions(-) diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 index 07f59d48..eb11e1a4 100644 --- a/ansible/templates/awxrestore_crd.yml.j2 +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -51,4 +51,4 @@ spec: description: Custom postgres_configuration secret name type: string oneOf: - - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"] + - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 3513684f..e303c137 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -496,7 +496,7 @@ spec: description: Custom postgres_configuration secret name type: string oneOf: - - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"] + - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml index 07f59d48..eb11e1a4 100644 --- a/deploy/crds/awxrestore_v1beta1_crd.yaml +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -51,4 +51,4 @@ spec: description: Custom postgres_configuration secret name type: string oneOf: - - required: ["tower_name", "tower_backup_pvc", "tower_backup_dir"] + - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] diff --git a/roles/backup/README.md b/roles/backup/README.md index 46215b85..509ffebe 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -10,8 +10,8 @@ The purpose of this role is to create a backup of your AWX deployment which incl Requirements ------------ -This role assumes you are authenticated with an Openshift or Kubernetes cluster which: - - The awx-operator has been deployed to +This role assumes you are authenticated with an Openshift or Kubernetes cluster: + - The awx-operator has been deployed to the cluster - AWX is deployed to via the operator @@ -60,7 +60,7 @@ tower_backup_storage_class: 'standard' tower_backup_size: '20Gi' ``` -By default, the backup pvc will be created in the `default` namespace. If you want your backup to be stored +By default, the backup pvc will be created in the same namespace the awxbackup object is created in. If you want your backup to be stored in a specific namespace, you can do so by specifying `tower_backup_pvc_namespace`. Keep in mind that you will need to provide the same namespace when restoring. @@ -68,12 +68,8 @@ need to provide the same namespace when restoring. tower_backup_pvc_namespace: 'custom-namespace' ``` -If a custom postgres configuration secret was used when deploying AWX, it must be set: - -``` -tower_postgres_configuration_secret: 'awx-postgres-configuration' -``` - +If a custom postgres configuration secret was used when deploying AWX, it will automatically be used by the backup role. +To check the name of this secret, look at the towerPostgresConfigurationSecret status on your AWX object. Testing ---------------- diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 2076c68c..d3e11cf2 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -4,16 +4,10 @@ tower_name: '' # Specify a pre-created PVC (name) to backup to tower_backup_pvc: '' -tower_backup_pvc_namespace: 'default' +tower_backup_pvc_namespace: "{{ meta.namespace }}" # Size of backup PVC if created dynamically tower_backup_size: '' # Specify storage class to determine how to dynamically create PVC's with tower_backup_storage_class: '' - -# Secret Names -tower_secret_key_secret: "{{ tower_name }}-secret-key" -tower_admin_password_secret: "{{ tower_name }}-admin-password" -tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket" -tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 69699796..35aa0a69 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -59,3 +59,16 @@ state: present template: "management-pod.yml.j2" wait: true + +# Retrieve AWX object for use in postgres.yml and secrets.yml +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + +- name: Look up details for this deployment + k8s_info: + api_version: "{{ api_version }}" + kind: "AWX" # Find a way to dynamically get this + name: "{{ tower_name }}" + namespace: "{{ meta.namespace }}" + register: this_awx diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index e9ca54ce..ddf9d0f4 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -1,23 +1,16 @@ --- -- name: Check for specified PostgreSQL configuration - k8s_info: - kind: Secret - namespace: '{{ meta.namespace }}' - name: '{{ tower_postgres_configuration_secret }}' - register: _custom_pg_config_resources - when: tower_postgres_configuration_secret | length - - name: Check for default PostgreSQL configuration k8s_info: kind: Secret namespace: '{{ meta.namespace }}' - name: '{{ tower_name }}-postgres-configuration' - register: _default_pg_config_resources + name: "{{ this_awx['resources'][0]['status']['towerPostgresConfigurationSecret'] }}" + register: pg_config -- name: Set PostgreSQL configuration - set_fact: - pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' +- name: Fail if postgres configuration secret status does not exist + fail: + msg: "The towerPostgresConfigurationSecret status is not set on the AWX object yet or the secret has been deleted." + when: not pg_config | default([]) | length - name: Store Database Configuration set_fact: diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 232c2ab8..5a60475c 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -1,17 +1,5 @@ --- -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - -- name: Look up details for this deployment - k8s_info: - api_version: "{{ api_version }}" - kind: "AWX" # Find a way to dynamically get this - name: "{{ tower_name }}" - namespace: "{{ meta.namespace }}" - register: this_awx - - name: Get secret_key k8s_info: kind: Secret diff --git a/roles/restore/README.md b/roles/restore/README.md index 0d9eb1fb..02eb8124 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -12,9 +12,10 @@ The purpose of this role is to restore your AWX deployment from an existing PVC Requirements ------------ -This role assumes you are authenticated with an Openshift or Kubernetes cluster which: - - The awx-operator has been deployed to +This role assumes you are authenticated with an Openshift or Kubernetes cluster: + - The awx-operator has been deployed to the cluster - AWX is deployed to via the operator + - An AWX backup is available on a PVC in your cluster (see the backup [README.md](../backup/README.md)) Usage @@ -32,6 +33,7 @@ metadata: spec: tower_name: mytower tower_backup_pvc: awxbackup1-backup-claim + tower_backup_pvc_namespace: 'old-awx-namespace' tower_backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 ``` @@ -78,6 +80,11 @@ awx-backup-volume-claim tower_backup_pvc: 'awx-backup-volume-claim' ``` +By default, the backup pvc will be created in the same namespace the awxbackup object is created in. This namespace must be specified using the `tower_backup_pvc_namespace` variable. + +``` +tower_backup_pvc_namespace: 'custom-namespace' +``` If a custom postgres configuration secret was used when deploying AWX, it must be set: diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml index 29a91b76..b99720be 100644 --- a/roles/restore/defaults/main.yml +++ b/roles/restore/defaults/main.yml @@ -4,7 +4,7 @@ tower_name: '' # Required: specify a pre-created PVC (name) to restore from tower_backup_pvc: '' -tower_backup_pvc_namespace: 'default' +tower_backup_pvc_namespace: '' # TODO: If the backup_dir is not provided, it should default to the most recent backup based on the timestamp at the end of the file name. # Required: backup name, found on the awxbackup object @@ -13,7 +13,10 @@ tower_backup_dir: '' # TODO: Should we add a unique id at the end of the secret when backing up, then use it here? # or will that make future backups more complicated because the user will have to specify the names of all the secrets? # Names of any secrets you want to use instead of the ones in the backup -tower_secret_key_secret: "{{ tower_name }}-secret-key" + +# TODO: Is this necessary? User's will be able to use the rekey role + tower_admin_password_secret: "{{ tower_name }}-admin-password" -tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket" tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration" +tower_secret_key_secret: "{{ tower_name }}-secret-key" +tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket" diff --git a/roles/restore/tasks/deploy_awx.yml b/roles/restore/tasks/deploy_awx.yml index 4332c5e3..5d59ef9a 100644 --- a/roles/restore/tasks/deploy_awx.yml +++ b/roles/restore/tasks/deploy_awx.yml @@ -15,7 +15,7 @@ - name: Deploy AWX k8s: state: "{{ state | default('present') }}" - namespace: "{{ meta.namespace | default('default') }}" + namespace: "{{ meta.namespace }}" apply: yes template: awx_object.yml.j2 wait: true diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index 5bde9fae..c45f7fed 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -74,9 +74,3 @@ when: - tower_backup_dir != '' - stat_backup_dir.return_code != 0 - -- name: Make temp definitions directory - tempfile: - prefix: "definitions-" - state: directory - register: definitions_dir diff --git a/roles/restore/tasks/secrets.yml b/roles/restore/tasks/secrets.yml index 1a9cd864..c703de04 100644 --- a/roles/restore/tasks/secrets.yml +++ b/roles/restore/tasks/secrets.yml @@ -31,7 +31,7 @@ - name: Apply secret k8s: state: present - namespace: "{{ meta.namespace | default('default') }}" + namespace: "{{ meta.namespace }}" apply: yes wait: yes template: "secrets.yml.j2" From 867bc258b96bb0adec1a85cb37c7999d333be735 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 22 Apr 2021 13:49:28 -0400 Subject: [PATCH 32/43] Allow custom postgres pod label to support user managed pg pods - Only set resolvable pg host path for pg container when managed --- ansible/templates/awxbackup_crd.yml.j2 | 4 ++++ ansible/templates/awxrestore_crd.yml.j2 | 3 +++ ansible/templates/crd.yml.j2 | 3 +++ deploy/awx-operator.yaml | 10 ++++++++++ deploy/crds/awx_v1beta1_crd.yaml | 3 +++ deploy/crds/awxbackup_v1beta1_crd.yaml | 4 ++++ deploy/crds/awxrestore_v1beta1_crd.yaml | 3 +++ docs/migration.md | 7 +++++-- roles/backup/README.md | 4 ++++ roles/backup/tasks/postgres.yml | 9 ++++++++- roles/installer/tasks/migrate_data.yml | 5 +++++ roles/restore/tasks/postgres.yml | 8 +++++++- 12 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ansible/templates/awxbackup_crd.yml.j2 b/ansible/templates/awxbackup_crd.yml.j2 index 02e18a27..21e388f3 100644 --- a/ansible/templates/awxbackup_crd.yml.j2 +++ b/ansible/templates/awxbackup_crd.yml.j2 @@ -53,5 +53,9 @@ spec: tower_postgres_configuration_secret: description: Custom postgres_configuration secret name type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + oneOf: - required: ["tower_name"] diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 index eb11e1a4..af08e551 100644 --- a/ansible/templates/awxrestore_crd.yml.j2 +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -50,5 +50,8 @@ spec: tower_postgres_configuration_secret: description: Custom postgres_configuration secret name type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string oneOf: - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] diff --git a/ansible/templates/crd.yml.j2 b/ansible/templates/crd.yml.j2 index c5956c1f..0bca0a1c 100644 --- a/ansible/templates/crd.yml.j2 +++ b/ansible/templates/crd.yml.j2 @@ -58,6 +58,9 @@ spec: tower_old_postgres_configuration_secret: description: Secret where the old database configuration can be found for data migration type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index e303c137..00c30c95 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -60,6 +60,9 @@ spec: tower_old_postgres_configuration_secret: description: Secret where the old database configuration can be found for data migration type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string @@ -440,6 +443,10 @@ spec: tower_postgres_configuration_secret: description: Custom postgres_configuration secret name type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + oneOf: - required: ["tower_name"] @@ -495,6 +502,9 @@ spec: tower_postgres_configuration_secret: description: Custom postgres_configuration secret name type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string oneOf: - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] diff --git a/deploy/crds/awx_v1beta1_crd.yaml b/deploy/crds/awx_v1beta1_crd.yaml index c5956c1f..0bca0a1c 100644 --- a/deploy/crds/awx_v1beta1_crd.yaml +++ b/deploy/crds/awx_v1beta1_crd.yaml @@ -58,6 +58,9 @@ spec: tower_old_postgres_configuration_secret: description: Secret where the old database configuration can be found for data migration type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string diff --git a/deploy/crds/awxbackup_v1beta1_crd.yaml b/deploy/crds/awxbackup_v1beta1_crd.yaml index 02e18a27..21e388f3 100644 --- a/deploy/crds/awxbackup_v1beta1_crd.yaml +++ b/deploy/crds/awxbackup_v1beta1_crd.yaml @@ -53,5 +53,9 @@ spec: tower_postgres_configuration_secret: description: Custom postgres_configuration secret name type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + oneOf: - required: ["tower_name"] diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml index eb11e1a4..af08e551 100644 --- a/deploy/crds/awxrestore_v1beta1_crd.yaml +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -50,5 +50,8 @@ spec: tower_postgres_configuration_secret: description: Custom postgres_configuration secret name type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string oneOf: - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] diff --git a/docs/migration.md b/docs/migration.md index f2b258d8..e83397e1 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -6,14 +6,14 @@ To migrate data from an older AWX installation, you must provide some informatio ### Secret Key -You can find your old secret key in the inventory file you used to deploy AWX in releases prior to version 18. +You can find your old secret key in the inventory file you used to deploy AWX in releases prior to version 18. ```yaml apiVersion: v1 kind: Secret metadata: name: -secret-key - namespace: + namespace: stringData: secret_key: type: Opaque @@ -49,6 +49,9 @@ In the next section pass it in through `tower_postgres_configuration_secret` ins from the key and ensuring the value matches the name of the secret. This will make AWX pick up on the existing database and apply any pending migrations. It is strongly recommended to backup your database beforehand. +The postgresql pod for the old deployment is used when streaming data to the new postgresql pod. If your postgresql pod has a custom label, +you can pass that via the `postgres_label_selector` variable to make sure the postgresql pod can be found. + ## Deploy AWX When you apply your AWX object, you must specify the name to the database secret you created above: diff --git a/roles/backup/README.md b/roles/backup/README.md index 509ffebe..5db6e418 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -71,6 +71,10 @@ tower_backup_pvc_namespace: 'custom-namespace' If a custom postgres configuration secret was used when deploying AWX, it will automatically be used by the backup role. To check the name of this secret, look at the towerPostgresConfigurationSecret status on your AWX object. +The postgresql pod for the old deployment is used when backing up data to the new postgresql pod. If your postgresql pod has a custom label, +you can pass that via the `postgres_label_selector` variable to make sure the postgresql pod can be found. + + Testing ---------------- diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index ddf9d0f4..4b891717 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -19,13 +19,19 @@ awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + awx_postgres_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" + +- name: Default label selector to custom resource generated postgres + set_fact: + postgres_label_selector: "app.kubernetes.io/name={{ meta.name }}-postgres" + when: postgres_label_selector is not defined - name: Get the postgres pod information k8s_info: kind: Pod namespace: '{{ meta.namespace }}' label_selectors: - - "app.kubernetes.io/name={{ tower_name }}-postgres" + - "{{ postgres_label_selector }}" register: postgres_pod until: - "postgres_pod['resources'] | length" @@ -69,6 +75,7 @@ - name: Set full resolvable host name for postgres pod set_fact: resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" + when: awx_postgres_type == 'managed' - name: Set pg_dump command set_fact: diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index e6bbab80..1f83d8f3 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -12,6 +12,11 @@ awx_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}" awx_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}" +- name: Default label selector to custom resource generated postgres + set_fact: + postgres_label_selector: "app.kubernetes.io/name={{ meta.name }}-postgres" + when: postgres_label_selector is not defined + - name: Get the postgres pod information k8s_info: kind: Pod diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index 13e38816..e64ee137 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -27,12 +27,17 @@ awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" +- name: Default label selector to custom resource generated postgres + set_fact: + postgres_label_selector: "app.kubernetes.io/name={{ meta.name }}-postgres" + when: postgres_label_selector is not defined + - name: Get the postgres pod information k8s_info: kind: Pod namespace: '{{ meta.namespace }}' label_selectors: - - "app.kubernetes.io/name={{ tower_name }}-postgres" + - "{{ postgres_label_selector }}" register: postgres_pod until: - "postgres_pod['resources'] | length" @@ -64,6 +69,7 @@ - name: Set full resolvable host name for postgres pod set_fact: resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" + when: awx_postgres_type == 'managed' - name: Set pg_restore command set_fact: From 3e444da7bc114e5903b615c4a02354a40e71de5a Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Thu, 22 Apr 2021 23:27:28 -0400 Subject: [PATCH 33/43] Set ownerRef to null for restore created AWX object to avoid garbage collection - Set defaults for pg type to satisfy conditional --- roles/backup/tasks/postgres.yml | 6 +++--- roles/restore/tasks/deploy_awx.yml | 10 ++++++++++ roles/restore/tasks/postgres.yml | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 4b891717..7aca192d 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -1,6 +1,6 @@ --- -- name: Check for default PostgreSQL configuration +- name: Get PostgreSQL configuration k8s_info: kind: Secret namespace: '{{ meta.namespace }}' @@ -19,11 +19,11 @@ awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" - awx_postgres_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" + awx_postgres_type: "{{ pg_config['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" - name: Default label selector to custom resource generated postgres set_fact: - postgres_label_selector: "app.kubernetes.io/name={{ meta.name }}-postgres" + postgres_label_selector: "app.kubernetes.io/name={{ tower_name }}-postgres" when: postgres_label_selector is not defined - name: Get the postgres pod information diff --git a/roles/restore/tasks/deploy_awx.yml b/roles/restore/tasks/deploy_awx.yml index 5d59ef9a..e333e7b6 100644 --- a/roles/restore/tasks/deploy_awx.yml +++ b/roles/restore/tasks/deploy_awx.yml @@ -26,3 +26,13 @@ # TODO: Add logic to allow users to provide override values here, # or to specify spec values that were not in the backed up AWX object. # This may involve changing how we back up the spec section of the AWX object + +- name: Remove ownerReferences to prevent garbage collection of new AWX CRO + k8s: + definition: + apiVersion: '{{ api_version }}' + kind: AWX + metadata: + name: '{{ tower_name }}' + namespace: '{{ meta.namespace }}' + ownerReferences: null diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index e64ee137..a8028d22 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -26,10 +26,11 @@ awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + awx_postgres_type: "{{ pg_config['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" - name: Default label selector to custom resource generated postgres set_fact: - postgres_label_selector: "app.kubernetes.io/name={{ meta.name }}-postgres" + postgres_label_selector: "app.kubernetes.io/name={{ tower_name }}-postgres" when: postgres_label_selector is not defined - name: Get the postgres pod information From 57f9530198045886f467e182bebcf2113e389699 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 23 Apr 2021 10:55:41 -0400 Subject: [PATCH 34/43] Simplify pvc naming scheme, one pvc per deployment --- roles/backup/tasks/init.yml | 2 +- roles/backup/templates/backup_pvc.yml.j2 | 2 +- roles/restore/README.md | 2 +- roles/restore/tasks/init.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index 35aa0a69..fbd00fc7 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -38,7 +38,7 @@ # If tower_backup_pvc is defined, use in management-pod.yml.j2 - name: Set default pvc name set_fact: - _default_backup_pvc: "{{ meta.name }}-backup-claim" + _default_backup_pvc: "{{ tower_name }}-backup-claim" # by default, it will re-use the old pvc if already created (unless a pvc is provided) - name: Set PVC to use for backup diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index 57778b82..ba0ae1a3 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ meta.name }}-backup-claim + name: {{ tower_name }}-backup-claim namespace: {{ tower_backup_pvc_namespace }} spec: accessModes: diff --git a/roles/restore/README.md b/roles/restore/README.md index 02eb8124..6439f3f6 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -32,7 +32,7 @@ metadata: namespace: my-namespace spec: tower_name: mytower - tower_backup_pvc: awxbackup1-backup-claim + tower_backup_pvc: myoldtower-backup-claim tower_backup_pvc_namespace: 'old-awx-namespace' tower_backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 ``` diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index c45f7fed..e40a926e 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -1,7 +1,7 @@ --- - name: Set default pvc name set_fact: - _default_backup_pvc: "{{ meta.name }}-backup-claim" + _default_backup_pvc: "{{ tower_name }}-backup-claim" # by default, it will re-use the old pvc if already created (unless pvc is provided) - name: Set PVC to use for backup From c817a2234d6c297e87e5dc09bfa7eed675f8fc00 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 23 Apr 2021 17:12:54 -0400 Subject: [PATCH 35/43] Simplify vars needed for restore CR & do not garbage collect secrets --- ansible/templates/awxrestore_crd.yml.j2 | 9 ++++--- deploy/awx-operator.yaml | 9 ++++--- deploy/crds/awxrestore_v1beta1_crd.yaml | 9 ++++--- roles/backup/README.md | 2 +- roles/restore/README.md | 10 +++++-- roles/restore/defaults/main.yml | 5 ---- roles/restore/tasks/cleanup.yml | 15 +++++++++++ roles/restore/tasks/init.yml | 26 ++++++++++++++----- roles/restore/templates/management-pod.yml.j2 | 2 +- 9 files changed, 62 insertions(+), 25 deletions(-) diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 index af08e551..db5c4e13 100644 --- a/ansible/templates/awxrestore_crd.yml.j2 +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -30,13 +30,16 @@ spec: description: Name of the deployment to be restored to type: string tower_backup_pvc: - description: Name of the PVC to be restored from + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (towerBackupClaim) type: string tower_backup_pvc_namespace: description: Namespace the PVC is in type: string tower_backup_dir: - description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) + description: Backup directory name, set as a status found on the awxbackup object (towerBackupDirectory) + type: string + tower_backup: + description: AWXBackup object name type: string tower_secret_key_secret: description: Custom secret_key secret name @@ -54,4 +57,4 @@ spec: description: Label selector used to identify postgres pod for backing up data type: string oneOf: - - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] + - required: ["tower_name", "tower_backup_pvc_namespace"] diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 00c30c95..67cfe045 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -482,13 +482,16 @@ spec: description: Name of the deployment to be restored to type: string tower_backup_pvc: - description: Name of the PVC to be restored from + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (towerBackupClaim) type: string tower_backup_pvc_namespace: description: Namespace the PVC is in type: string tower_backup_dir: - description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) + description: Backup directory name, set as a status found on the awxbackup object (towerBackupDirectory) + type: string + tower_backup: + description: AWXBackup object name type: string tower_secret_key_secret: description: Custom secret_key secret name @@ -506,7 +509,7 @@ spec: description: Label selector used to identify postgres pod for backing up data type: string oneOf: - - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] + - required: ["tower_name", "tower_backup_pvc_namespace"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml index af08e551..db5c4e13 100644 --- a/deploy/crds/awxrestore_v1beta1_crd.yaml +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -30,13 +30,16 @@ spec: description: Name of the deployment to be restored to type: string tower_backup_pvc: - description: Name of the PVC to be restored from + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (towerBackupClaim) type: string tower_backup_pvc_namespace: description: Namespace the PVC is in type: string tower_backup_dir: - description: Backup directory name, a status found on the awxbackup object (towerBackupComplete) + description: Backup directory name, set as a status found on the awxbackup object (towerBackupDirectory) + type: string + tower_backup: + description: AWXBackup object name type: string tower_secret_key_secret: description: Custom secret_key secret name @@ -54,4 +57,4 @@ spec: description: Label selector used to identify postgres pod for backing up data type: string oneOf: - - required: ["tower_name", "tower_backup_pvc", "tower_backup_pvc_namespace", "tower_backup_dir"] + - required: ["tower_name", "tower_backup_pvc_namespace"] diff --git a/roles/backup/README.md b/roles/backup/README.md index 5db6e418..e9e8b599 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -25,7 +25,7 @@ Then create a file named `backup-awx.yml` with the following contents: apiVersion: awx.ansible.com/v1beta1 kind: AWXBackup metadata: - name: awxbackup + name: awxbackup-2021-04-22 namespace: my-namespace spec: tower_name: mytower diff --git a/roles/restore/README.md b/roles/restore/README.md index 6439f3f6..be2db0a4 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -32,9 +32,8 @@ metadata: namespace: my-namespace spec: tower_name: mytower - tower_backup_pvc: myoldtower-backup-claim + tower_backup: awxbackup-2021-04-22 tower_backup_pvc_namespace: 'old-awx-namespace' - tower_backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 ``` Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. @@ -92,6 +91,13 @@ If a custom postgres configuration secret was used when deploying AWX, it must b tower_postgres_configuration_secret: 'awx-postgres-configuration' ``` +If the awxbackup object no longer exists, it is still possible to restore from the backup it created by specifying the pvc name and the back directory. + +``` +tower_backup_pvc: myoldtower-backup-claim +tower_backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 +``` + Testing ---------------- diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml index b99720be..f581f34c 100644 --- a/roles/restore/defaults/main.yml +++ b/roles/restore/defaults/main.yml @@ -6,14 +6,9 @@ tower_name: '' tower_backup_pvc: '' tower_backup_pvc_namespace: '' -# TODO: If the backup_dir is not provided, it should default to the most recent backup based on the timestamp at the end of the file name. # Required: backup name, found on the awxbackup object tower_backup_dir: '' -# TODO: Should we add a unique id at the end of the secret when backing up, then use it here? -# or will that make future backups more complicated because the user will have to specify the names of all the secrets? -# Names of any secrets you want to use instead of the ones in the backup - # TODO: Is this necessary? User's will be able to use the rekey role tower_admin_password_secret: "{{ tower_name }}-admin-password" diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml index 7e7e451b..2d66da16 100644 --- a/roles/restore/tasks/cleanup.yml +++ b/roles/restore/tasks/cleanup.yml @@ -7,3 +7,18 @@ namespace: "{{ tower_backup_pvc_namespace }}" state: absent force: true + +- name: Remove ownerReferences from secrets to avoid garbage collection + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: '{{ item }}' + namespace: '{{ meta.namespace }}' + ownerReferences: null + loop: + - '{{ tower_name }}-admin-password' + - '{{ tower_name }}-secret-key' + - '{{ tower_name }}-postgres-configuration' + - '{{ tower_name }}-broadcast-websocket' diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index e40a926e..bb6247b9 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -1,12 +1,24 @@ --- -- name: Set default pvc name - set_fact: - _default_backup_pvc: "{{ tower_name }}-backup-claim" -# by default, it will re-use the old pvc if already created (unless pvc is provided) -- name: Set PVC to use for backup - set_fact: - backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" +- name: Set variables from awxbackup object statuses if provided + block: + - name: Look up details for the backup object + k8s_info: + api_version: "{{ api_version }}" + kind: "AWXBackup" + name: "{{ tower_backup }}" + namespace: "{{ tower_backup_pvc_namespace }}" + register: this_backup + + - name: Set backup pvc name from status + set_fact: + tower_backup_pvc: "{{ this_backup['resources'][0]['status']['towerBackupClaim'] }}" + + - name: Set tmp backup directory from status + set_fact: + tower_backup_dir: "{{ this_backup['resources'][0]['status']['towerBackupDirectory'] }}" + when: + - tower_backup != '' or tower_backup is defined # Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. - name: Check provided PVC exists diff --git a/roles/restore/templates/management-pod.yml.j2 b/roles/restore/templates/management-pod.yml.j2 index d938da20..a60e944d 100644 --- a/roles/restore/templates/management-pod.yml.j2 +++ b/roles/restore/templates/management-pod.yml.j2 @@ -17,6 +17,6 @@ spec: volumes: - name: {{ meta.name }}-backup persistentVolumeClaim: - claimName: {{ backup_pvc }} + claimName: {{ tower_backup_pvc }} readOnly: false restartPolicy: Never From d743936ee479ccf816eb11cba2a83571be1ef716 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 27 Apr 2021 10:43:02 -0400 Subject: [PATCH 36/43] Update admin user password with value in provided/generated secret --- roles/installer/tasks/initialize_django.yml | 13 ++++- roles/restore/README.md | 4 +- roles/restore/tasks/initialize_django.yml | 54 +++++++++++++++++++++ roles/restore/tasks/main.yml | 2 + 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 roles/restore/tasks/initialize_django.yml diff --git a/roles/installer/tasks/initialize_django.yml b/roles/installer/tasks/initialize_django.yml index cb6a6e1b..3e977e06 100644 --- a/roles/installer/tasks/initialize_django.yml +++ b/roles/installer/tasks/initialize_django.yml @@ -6,13 +6,24 @@ container: "{{ meta.name }}-task" command: >- bash -c "echo 'from django.contrib.auth.models import User; - nsu = User.objects.filter(is_superuser=True).count(); + nsu = User.objects.filter(is_superuser=True, username='{{ tower_admin_user }}').count(); exit(0 if nsu > 0 else 1)' | awx-manage shell" ignore_errors: true register: users_result changed_when: users_result.return_code > 0 +- name: Update super user password via Django if it does exist (same password is a noop) + k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ tower_pod_name }}" + container: "{{ meta.name }}-task" + command: >- + bash -c "awx-manage update_password --username '{{ tower_admin_user }}' --password '{{ tower_admin_password }}'" + register: update_pw_result + changed_when: users_result.stdout == 'Password not updated' + when: users_result.return_code == 0 + - name: Create super user via Django if it doesn't exist. k8s_exec: namespace: "{{ meta.namespace }}" diff --git a/roles/restore/README.md b/roles/restore/README.md index be2db0a4..e2b31378 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -50,7 +50,9 @@ Finally, use `kubectl` to create the restore object in your cluster: $ kubectl apply -f restore-awx.yml ``` -This will create a new deployment and restore your backup to it. +This will create a new deployment and restore your backup to it. + +> :warning: tower_admin_password_secret value will replace the password for the `tower_admin_user` user (by default, this is the `admin` user). Role Variables diff --git a/roles/restore/tasks/initialize_django.yml b/roles/restore/tasks/initialize_django.yml new file mode 100644 index 00000000..e1c350cd --- /dev/null +++ b/roles/restore/tasks/initialize_django.yml @@ -0,0 +1,54 @@ +--- + +- name: Get the new deployment resource pod information. + k8s_info: + api_version: v1 + kind: Pod + namespace: '{{ tower_backup_pvc_namespace }}' + label_selectors: + - "app.kubernetes.io/name={{ tower_name }}" + - "app.kubernetes.io/managed-by=awx-operator" + - "app.kubernetes.io/component=awx" + field_selectors: + - status.phase=Running + register: tower_pods + +- name: Set the resource pod name as a variable. + set_fact: + tower_pod_name: "{{ tower_pods['resources'][0]['metadata']['name'] | default('') }}" + +- name: Check if there are any super users defined. + k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ tower_pod_name }}" + container: "{{ meta.name }}-task" + command: >- + bash -c "echo 'from django.contrib.auth.models import User; + nsu = User.objects.filter(is_superuser=True, username='{{ tower_admin_user }}').count(); + exit(0 if nsu > 0 else 1)' + | awx-manage shell" + ignore_errors: true + register: users_result + changed_when: users_result.return_code > 0 + +- name: Update super user password via Django if it does exist (same password is a noop) + k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ tower_pod_name }}" + container: "{{ meta.name }}-task" + command: >- + bash -c "awx-manage update_password --username '{{ tower_admin_user }}' --password '{{ tower_admin_password }}'" + register: update_pw_result + changed_when: users_result.stdout == 'Password not updated' + when: users_result.return_code == 0 + +- name: Create super user via Django if it doesn't exist. + k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ tower_pod_name }}" + container: "{{ meta.name }}-task" + command: >- + bash -c "echo \"from django.contrib.auth.models import User; + User.objects.create_superuser('{{ tower_admin_user }}', '{{ tower_admin_email }}', '{{ tower_admin_password }}')\" + | awx-manage shell" + when: users_result.return_code > 0 diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index 7fda2e1c..5d861909 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -22,6 +22,8 @@ - include_tasks: postgres.yml + - include_tasks: initialize_django.yml + - name: Set flag signifying this restore was successful set_fact: tower_restore_complete: True From 5ae36367a4803f9b4ba6550d7a70ae8de44b2ec6 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Tue, 27 Apr 2021 22:51:29 -0400 Subject: [PATCH 37/43] Rename product specific variable names --- README.md | 2 +- ansible/templates/awxbackup_crd.yml.j2 | 14 ++++----- ansible/templates/awxrestore_crd.yml.j2 | 20 ++++++------- roles/backup/README.md | 14 ++++----- roles/backup/defaults/main.yml | 10 +++---- roles/backup/tasks/awx-cro.yml | 4 +-- roles/backup/tasks/cleanup.yml | 2 +- roles/backup/tasks/init.yml | 24 +++++++-------- roles/backup/tasks/main.yml | 4 +-- roles/backup/tasks/postgres.yml | 10 +++---- roles/backup/tasks/secrets.yml | 2 +- roles/backup/tasks/update_status.yml | 6 ++-- roles/backup/templates/backup_pvc.yml.j2 | 10 +++---- roles/backup/templates/management-pod.yml.j2 | 6 ++-- roles/backup/vars/main.yml | 4 +-- roles/restore/README.md | 24 +++++++-------- roles/restore/defaults/main.yml | 16 +++++----- roles/restore/tasks/cleanup.yml | 10 +++---- roles/restore/tasks/deploy_awx.yml | 6 ++-- roles/restore/tasks/init.yml | 30 +++++++++---------- roles/restore/tasks/initialize_django.yml | 4 +-- roles/restore/tasks/postgres.yml | 8 ++--- roles/restore/tasks/secrets.yml | 8 ++--- roles/restore/templates/awx_object.yml.j2 | 2 +- roles/restore/templates/management-pod.yml.j2 | 6 ++-- roles/restore/templates/secrets.yml.j2 | 8 ++--- roles/restore/vars/main.yml | 2 +- 27 files changed, 128 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index aede3d3e..0a0b73e5 100644 --- a/README.md +++ b/README.md @@ -664,7 +664,7 @@ After it is built, test it on a local cluster: #> minikube addons enable ingress #> ansible-playbook ansible/deploy-operator.yml -e operator_image=quay.io//awx-operator -e operator_version=test #> kubectl create namespace example-awx -#> ansible-playbook ansible/instantiate-awx-deployment.yml -e tower_namespace=example-awx +#> ansible-playbook ansible/instantiate-awx-deployment.yml -e namespace=example-awx #> #> minikube delete ``` diff --git a/ansible/templates/awxbackup_crd.yml.j2 b/ansible/templates/awxbackup_crd.yml.j2 index 21e388f3..fd5f04be 100644 --- a/ansible/templates/awxbackup_crd.yml.j2 +++ b/ansible/templates/awxbackup_crd.yml.j2 @@ -26,19 +26,19 @@ spec: spec: type: object properties: - tower_name: + deployment_name: description: Name of the deployment to be backed up type: string - tower_backup_pvc: + backup_pvc: description: Name of the PVC to be used for storing the backup type: string - tower_backup_pvc_namespace: + backup_pvc_namespace: description: Namespace PVC is in type: string - tower_backup_size: - description: Size of PVC + backup_storage_requirements: + description: Storage requirements for the PostgreSQL container type: string - tower_backup_storage_class: + backup_storage_class: description: Storage class to use when creating PVC for backup type: string tower_secret_key_secret: @@ -58,4 +58,4 @@ spec: type: string oneOf: - - required: ["tower_name"] + - required: ["deployment_name"] diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 index db5c4e13..8a91ce07 100644 --- a/ansible/templates/awxrestore_crd.yml.j2 +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -26,20 +26,20 @@ spec: spec: type: object properties: - tower_name: + deployment_name: description: Name of the deployment to be restored to type: string - tower_backup_pvc: - description: Name of the PVC to be restored from, set as a status found on the awxbackup object (towerBackupClaim) + backup: + description: AWXBackup object name type: string - tower_backup_pvc_namespace: + backup_pvc: + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim) + type: string + backup_pvc_namespace: description: Namespace the PVC is in type: string - tower_backup_dir: - description: Backup directory name, set as a status found on the awxbackup object (towerBackupDirectory) - type: string - tower_backup: - description: AWXBackup object name + backup_dir: + description: Backup directory name, set as a status found on the awxbackup object (backupDirectory) type: string tower_secret_key_secret: description: Custom secret_key secret name @@ -57,4 +57,4 @@ spec: description: Label selector used to identify postgres pod for backing up data type: string oneOf: - - required: ["tower_name", "tower_backup_pvc_namespace"] + - required: ["deployment_name", "backup_pvc_namespace"] diff --git a/roles/backup/README.md b/roles/backup/README.md index e9e8b599..3f482f72 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -28,10 +28,10 @@ metadata: name: awxbackup-2021-04-22 namespace: my-namespace spec: - tower_name: mytower + deployment_name: mytower ``` -Note that the `tower_name` above is the name of the AWX deployment you intend to backup from. The namespace above is the one containing the AWX deployment that will be backed up. +Note that the `deployment_name` above is the name of the AWX deployment you intend to backup from. The namespace above is the one containing the AWX deployment that will be backed up. Finally, use `kubectl` to create the backup object in your cluster: @@ -48,7 +48,7 @@ Role Variables A custom, pre-created pvc can be used by setting the following variables. ``` -tower_backup_pvc: 'awx-backup-volume-claim' +backup_pvc: 'awx-backup-volume-claim' ``` > If no pvc or storage class is provided, the cluster's default storage class will be used to create the pvc. @@ -56,16 +56,16 @@ tower_backup_pvc: 'awx-backup-volume-claim' This role will automatically create a pvc using a Storage Class if provided: ``` -tower_backup_storage_class: 'standard' -tower_backup_size: '20Gi' +backup_storage_class: 'standard' +backup_storage_requirements: '20Gi' ``` By default, the backup pvc will be created in the same namespace the awxbackup object is created in. If you want your backup to be stored -in a specific namespace, you can do so by specifying `tower_backup_pvc_namespace`. Keep in mind that you will +in a specific namespace, you can do so by specifying `backup_pvc_namespace`. Keep in mind that you will need to provide the same namespace when restoring. ``` -tower_backup_pvc_namespace: 'custom-namespace' +backup_pvc_namespace: 'custom-namespace' ``` If a custom postgres configuration secret was used when deploying AWX, it will automatically be used by the backup role. diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index d3e11cf2..484b54f2 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,13 +1,13 @@ --- # Required: specify name of tower deployment to backup from -tower_name: '' +deployment_name: '' # Specify a pre-created PVC (name) to backup to -tower_backup_pvc: '' -tower_backup_pvc_namespace: "{{ meta.namespace }}" +backup_pvc: '' +backup_pvc_namespace: "{{ meta.namespace }}" # Size of backup PVC if created dynamically -tower_backup_size: '' +backup_storage_requirements: '' # Specify storage class to determine how to dynamically create PVC's with -tower_backup_storage_class: '' +backup_storage_class: '' diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index 3587ce70..3e487dd9 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -5,7 +5,7 @@ version: v1beta1 kind: AWX namespace: '{{ meta.namespace }}' - name: '{{ tower_name }}' + name: '{{ deployment_name }}' register: _awx_cro - name: Set AWX object @@ -22,7 +22,7 @@ - name: Write awx object to pvc k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "echo '{{ awx_definition_file }}' > {{ backup_dir }}/awx_object" diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml index 7e7e451b..f91c8e6c 100644 --- a/roles/backup/tasks/cleanup.yml +++ b/roles/backup/tasks/cleanup.yml @@ -4,6 +4,6 @@ k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" state: absent force: true diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index fbd00fc7..a0d208e6 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -4,7 +4,7 @@ k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" state: absent force: true wait: true @@ -12,45 +12,45 @@ # Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. - name: Check provided PVC exists k8s_info: - name: "{{ tower_backup_pvc }}" + name: "{{ backup_pvc }}" kind: PersistentVolumeClaim - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" register: provided_pvc when: - - tower_backup_pvc != '' + - backup_pvc != '' - name: Surface error to user block: - name: Set error message set_fact: - error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + error_msg: "{{ backup_pvc }} does not exist, please create this pvc first." - name: Handle error import_tasks: error_handling.yml - name: Fail early if pvc is defined but does not exist fail: - msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + msg: "{{ backup_pvc }} does not exist, please create this pvc first." when: - - tower_backup_pvc != '' + - backup_pvc != '' - provided_pvc.resources | length == 0 -# If tower_backup_pvc is defined, use in management-pod.yml.j2 +# If backup_pvc is defined, use in management-pod.yml.j2 - name: Set default pvc name set_fact: - _default_backup_pvc: "{{ tower_name }}-backup-claim" + _default_backup_pvc: "{{ deployment_name }}-backup-claim" # by default, it will re-use the old pvc if already created (unless a pvc is provided) - name: Set PVC to use for backup set_fact: - backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}" + backup_claim: "{{ backup_pvc | default(_default_backup_pvc, true) }}" - name: Create PVC for backup k8s: kind: PersistentVolumeClaim template: "backup_pvc.yml.j2" when: - - tower_backup_pvc == '' or tower_backup_pvc is not defined + - backup_pvc == '' or backup_pvc is not defined - name: Create management pod from templated deployment config k8s: @@ -69,6 +69,6 @@ k8s_info: api_version: "{{ api_version }}" kind: "AWX" # Find a way to dynamically get this - name: "{{ tower_name }}" + name: "{{ deployment_name }}" namespace: "{{ meta.namespace }}" register: this_awx diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index 0966577f..f8203871 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -24,12 +24,12 @@ - name: Set flag signifying this backup was successful set_fact: - tower_backup_complete: true + backup_complete: true - include_tasks: cleanup.yml when: - - this_backup['resources'][0]['status']['towerBackupDirectory'] is not defined + - this_backup['resources'][0]['status']['backupDirectory'] is not defined - name: Update status variables include_tasks: update_status.yml diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml index 7aca192d..96e8c588 100644 --- a/roles/backup/tasks/postgres.yml +++ b/roles/backup/tasks/postgres.yml @@ -23,7 +23,7 @@ - name: Default label selector to custom resource generated postgres set_fact: - postgres_label_selector: "app.kubernetes.io/name={{ tower_name }}-postgres" + postgres_label_selector: "app.kubernetes.io/name={{ deployment_name }}-postgres" when: postgres_label_selector is not defined - name: Get the postgres pod information @@ -53,21 +53,21 @@ - name: Create directory for backup k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- mkdir -p {{ backup_dir }} - name: Precreate file for database dump k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- touch {{ backup_dir }}/tower.db - name: Set permissions on file for database dump k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db" @@ -88,7 +88,7 @@ - name: Write pg_dump to backup on PVC k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ backup_dir }}/tower.db" diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index 5a60475c..a93483e4 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -55,7 +55,7 @@ - name: Write postgres configuration to pvc k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- bash -c "echo '{{ secrets_file }}' > {{ backup_dir }}/secrets.yml" diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml index b2894f22..fe55aa92 100644 --- a/roles/backup/tasks/update_status.yml +++ b/roles/backup/tasks/update_status.yml @@ -12,6 +12,6 @@ name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" status: - towerBackupDirectory: "{{ backup_dir }}" - towerBackupClaim: "{{ backup_pvc }}" - when: tower_backup_complete + backupDirectory: "{{ backup_dir }}" + backupClaim: "{{ backup_claim }}" + when: backup_complete diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 index ba0ae1a3..d024a565 100644 --- a/roles/backup/templates/backup_pvc.yml.j2 +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -2,14 +2,14 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ tower_name }}-backup-claim - namespace: {{ tower_backup_pvc_namespace }} + name: {{ deployment_name }}-backup-claim + namespace: {{ backup_pvc_namespace }} spec: accessModes: - ReadWriteOnce -{% if tower_backup_storage_class != '' %} - storageClassName: {{ tower_backup_storage_class }} +{% if backup_storage_class != '' %} + storageClassName: {{ backup_storage_class }} {% endif %} resources: requests: - storage: {{ tower_backup_size | default('5Gi', true) }} + storage: {{ backup_storage_requirements | default('5Gi', true) }} diff --git a/roles/backup/templates/management-pod.yml.j2 b/roles/backup/templates/management-pod.yml.j2 index d938da20..9aa99a1c 100644 --- a/roles/backup/templates/management-pod.yml.j2 +++ b/roles/backup/templates/management-pod.yml.j2 @@ -3,11 +3,11 @@ apiVersion: v1 kind: Pod metadata: name: {{ meta.name }}-db-management - namespace: {{ tower_backup_pvc_namespace }} + namespace: {{ backup_pvc_namespace }} spec: containers: - name: {{ meta.name }}-db-management - image: "{{ tower_postgres_image }}" + image: "{{ postgres_image }}" imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: @@ -17,6 +17,6 @@ spec: volumes: - name: {{ meta.name }}-backup persistentVolumeClaim: - claimName: {{ backup_pvc }} + claimName: {{ backup_claim }} readOnly: false restartPolicy: Never diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index f1dad4a2..acf8ceef 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -1,4 +1,4 @@ --- deployment_type: "awx" -tower_postgres_image: postgres:12 -tower_backup_complete: false +postgres_image: postgres:12 +backup_complete: false diff --git a/roles/restore/README.md b/roles/restore/README.md index e2b31378..0ba8580b 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -31,12 +31,12 @@ metadata: name: restore1 namespace: my-namespace spec: - tower_name: mytower - tower_backup: awxbackup-2021-04-22 - tower_backup_pvc_namespace: 'old-awx-namespace' + deployment_name: mytower + backup: awxbackup-2021-04-22 + backup_pvc_namespace: 'old-awx-namespace' ``` -Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to. +Note that the `deployment_name` above is the name of the AWX deployment you intend to create and restore to. The namespace specified is the namespace the resulting AWX deployment will be in. The namespace you specified must be pre-created. @@ -61,30 +61,30 @@ Role Variables The name of the backup directory can be found as a status on your AWXBackup object. This can be found in your cluster's console, or with the client as shown below. ```bash -$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.towerBackupDirectory}" +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.backupDirectory}" /backups/tower-openshift-backup-2021-04-02-03:25:08 ``` ``` -tower_backup_dir: '/backups/tower-openshift-backup-2021-04-02-03:25:08' +backup_dir: '/backups/tower-openshift-backup-2021-04-02-03:25:08' ``` The name of the PVC can also be found by looking at the backup object. ```bash -$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.towerBackupClaim}" +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.backupClaim}" awx-backup-volume-claim ``` ``` -tower_backup_pvc: 'awx-backup-volume-claim' +backup_pvc: 'awx-backup-volume-claim' ``` -By default, the backup pvc will be created in the same namespace the awxbackup object is created in. This namespace must be specified using the `tower_backup_pvc_namespace` variable. +By default, the backup pvc will be created in the same namespace the awxbackup object is created in. This namespace must be specified using the `backup_pvc_namespace` variable. ``` -tower_backup_pvc_namespace: 'custom-namespace' +backup_pvc_namespace: 'custom-namespace' ``` If a custom postgres configuration secret was used when deploying AWX, it must be set: @@ -96,8 +96,8 @@ tower_postgres_configuration_secret: 'awx-postgres-configuration' If the awxbackup object no longer exists, it is still possible to restore from the backup it created by specifying the pvc name and the back directory. ``` -tower_backup_pvc: myoldtower-backup-claim -tower_backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 +backup_pvc: myoldtower-backup-claim +backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 ``` diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml index f581f34c..387015d5 100644 --- a/roles/restore/defaults/main.yml +++ b/roles/restore/defaults/main.yml @@ -1,17 +1,17 @@ --- # Required: specify name of tower deployment to restore to -tower_name: '' +deployment_name: '' # Required: specify a pre-created PVC (name) to restore from -tower_backup_pvc: '' -tower_backup_pvc_namespace: '' +backup_pvc: '' +backup_pvc_namespace: '' # Required: backup name, found on the awxbackup object -tower_backup_dir: '' +backup_dir: '' # TODO: Is this necessary? User's will be able to use the rekey role -tower_admin_password_secret: "{{ tower_name }}-admin-password" -tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration" -tower_secret_key_secret: "{{ tower_name }}-secret-key" -tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket" +tower_admin_password_secret: "{{ deployment_name }}-admin-password" +tower_postgres_configuration_secret: "{{ deployment_name }}-postgres-configuration" +tower_secret_key_secret: "{{ deployment_name }}-secret-key" +tower_broadcast_websocket_secret: "{{ deployment_name }}-broadcast-websocket" diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml index 2d66da16..94a545be 100644 --- a/roles/restore/tasks/cleanup.yml +++ b/roles/restore/tasks/cleanup.yml @@ -4,7 +4,7 @@ k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" state: absent force: true @@ -18,7 +18,7 @@ namespace: '{{ meta.namespace }}' ownerReferences: null loop: - - '{{ tower_name }}-admin-password' - - '{{ tower_name }}-secret-key' - - '{{ tower_name }}-postgres-configuration' - - '{{ tower_name }}-broadcast-websocket' + - '{{ deployment_name }}-admin-password' + - '{{ deployment_name }}-secret-key' + - '{{ deployment_name }}-postgres-configuration' + - '{{ deployment_name }}-broadcast-websocket' diff --git a/roles/restore/tasks/deploy_awx.yml b/roles/restore/tasks/deploy_awx.yml index e333e7b6..5fc1b7fc 100644 --- a/roles/restore/tasks/deploy_awx.yml +++ b/roles/restore/tasks/deploy_awx.yml @@ -2,10 +2,10 @@ - name: Get AWX object definition from pvc k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "cat '{{ tower_backup_dir }}/awx_object'" + bash -c "cat '{{ backup_dir }}/awx_object'" register: awx_object - name: Set AWX spec variable from backup @@ -33,6 +33,6 @@ apiVersion: '{{ api_version }}' kind: AWX metadata: - name: '{{ tower_name }}' + name: '{{ deployment_name }}' namespace: '{{ meta.namespace }}' ownerReferences: null diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml index bb6247b9..9b16e9cd 100644 --- a/roles/restore/tasks/init.yml +++ b/roles/restore/tasks/init.yml @@ -6,35 +6,35 @@ k8s_info: api_version: "{{ api_version }}" kind: "AWXBackup" - name: "{{ tower_backup }}" - namespace: "{{ tower_backup_pvc_namespace }}" + name: "{{ backup }}" + namespace: "{{ backup_pvc_namespace }}" register: this_backup - name: Set backup pvc name from status set_fact: - tower_backup_pvc: "{{ this_backup['resources'][0]['status']['towerBackupClaim'] }}" + backup_pvc: "{{ this_backup['resources'][0]['status']['backupClaim'] }}" - name: Set tmp backup directory from status set_fact: - tower_backup_dir: "{{ this_backup['resources'][0]['status']['towerBackupDirectory'] }}" + backup_dir: "{{ this_backup['resources'][0]['status']['backupDirectory'] }}" when: - - tower_backup != '' or tower_backup is defined + - backup != '' or backup is defined # Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. - name: Check provided PVC exists k8s_info: - name: "{{ tower_backup_pvc }}" + name: "{{ backup_pvc }}" kind: PersistentVolumeClaim - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" register: provided_pvc when: - - tower_backup_pvc != '' + - backup_pvc != '' - name: Surface error to user block: - name: Set error message set_fact: - error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first." + error_msg: "{{ backup_pvc }} does not exist, please create this pvc first." - name: Handle error import_tasks: error_handling.yml @@ -43,14 +43,14 @@ fail: msg: "{{ error_msg }}" when: - - tower_backup_pvc != '' + - backup_pvc != '' - provided_pvc.resources | length == 0 - name: Delete any existing management pod k8s: name: "{{ meta.name }}-db-management" kind: Pod - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" state: absent force: true wait: true @@ -65,17 +65,17 @@ - name: Check to make sure backup directory exists on PVC k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "stat {{ tower_backup_dir }}" + bash -c "stat {{ backup_dir }}" register: stat_backup_dir - name: Error if backup dir is missing block: - name: Set error message set_fact: - error_msg: "{{ tower_backup_dir }} does not exist, see the towerBackupDirectory status on your AWXBackup for the correct tower_backup_dir." + error_msg: "{{ backup_dir }} does not exist, see the backupDirectory status on your AWXBackup for the correct backup_dir." - name: Handle error import_tasks: error_handling.yml @@ -84,5 +84,5 @@ fail: msg: "{{ error_msg }}" when: - - tower_backup_dir != '' + - backup_dir != '' - stat_backup_dir.return_code != 0 diff --git a/roles/restore/tasks/initialize_django.yml b/roles/restore/tasks/initialize_django.yml index e1c350cd..f1f217fe 100644 --- a/roles/restore/tasks/initialize_django.yml +++ b/roles/restore/tasks/initialize_django.yml @@ -4,9 +4,9 @@ k8s_info: api_version: v1 kind: Pod - namespace: '{{ tower_backup_pvc_namespace }}' + namespace: '{{ backup_pvc_namespace }}' label_selectors: - - "app.kubernetes.io/name={{ tower_name }}" + - "app.kubernetes.io/name={{ deployment_name }}" - "app.kubernetes.io/managed-by=awx-operator" - "app.kubernetes.io/component=awx" field_selectors: diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml index a8028d22..967cd77c 100644 --- a/roles/restore/tasks/postgres.yml +++ b/roles/restore/tasks/postgres.yml @@ -12,7 +12,7 @@ k8s_info: kind: Secret namespace: '{{ meta.namespace }}' - name: '{{ tower_name }}-postgres-configuration' + name: '{{ deployment_name }}-postgres-configuration' register: _default_pg_config_resources - name: Set PostgreSQL configuration @@ -30,7 +30,7 @@ - name: Default label selector to custom resource generated postgres set_fact: - postgres_label_selector: "app.kubernetes.io/name={{ tower_name }}-postgres" + postgres_label_selector: "app.kubernetes.io/name={{ deployment_name }}-postgres" when: postgres_label_selector is not defined - name: Get the postgres pod information @@ -83,12 +83,12 @@ - name: Restore database dump to the new postgresql container k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: | bash -c """ set -e -o pipefail - cat {{ tower_backup_dir }}/tower.db | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} + cat {{ backup_dir }}/tower.db | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} echo 'Successful' """ register: data_migration diff --git a/roles/restore/tasks/secrets.yml b/roles/restore/tasks/secrets.yml index c703de04..b543769a 100644 --- a/roles/restore/tasks/secrets.yml +++ b/roles/restore/tasks/secrets.yml @@ -2,10 +2,10 @@ - name: Get secret definition from pvc k8s_exec: - namespace: "{{ tower_backup_pvc_namespace }}" + namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "cat '{{ tower_backup_dir }}/secrets.yml'" + bash -c "cat '{{ backup_dir }}/secrets.yml'" register: secrets - name: Create temp vars file @@ -22,9 +22,9 @@ - name: Include secret vars from backup include_vars: "{{ secret_vars.path }}" -- name: Set new database host based on supplied tower_name +- name: Set new database host based on supplied deployment_name set_fact: - database_host: "{{ tower_name }}-postgres" + database_host: "{{ deployment_name }}-postgres" when: - database_type == 'managed' diff --git a/roles/restore/templates/awx_object.yml.j2 b/roles/restore/templates/awx_object.yml.j2 index b13e3a51..53f01326 100644 --- a/roles/restore/templates/awx_object.yml.j2 +++ b/roles/restore/templates/awx_object.yml.j2 @@ -2,6 +2,6 @@ apiVersion: '{{ api_version }}' kind: AWX metadata: - name: '{{ tower_name }}' + name: '{{ deployment_name }}' namespace: '{{ meta.namespace }}' spec: {{ awx_spec }} diff --git a/roles/restore/templates/management-pod.yml.j2 b/roles/restore/templates/management-pod.yml.j2 index a60e944d..301bbfbb 100644 --- a/roles/restore/templates/management-pod.yml.j2 +++ b/roles/restore/templates/management-pod.yml.j2 @@ -3,11 +3,11 @@ apiVersion: v1 kind: Pod metadata: name: {{ meta.name }}-db-management - namespace: {{ tower_backup_pvc_namespace }} + namespace: {{ backup_pvc_namespace }} spec: containers: - name: {{ meta.name }}-db-management - image: "{{ tower_postgres_image }}" + image: "{{ postgres_image }}" imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: @@ -17,6 +17,6 @@ spec: volumes: - name: {{ meta.name }}-backup persistentVolumeClaim: - claimName: {{ tower_backup_pvc }} + claimName: {{ backup_pvc }} readOnly: false restartPolicy: Never diff --git a/roles/restore/templates/secrets.yml.j2 b/roles/restore/templates/secrets.yml.j2 index 39bca5c8..4d718421 100644 --- a/roles/restore/templates/secrets.yml.j2 +++ b/roles/restore/templates/secrets.yml.j2 @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: - name: '{{ tower_name }}-postgres-configuration' + name: '{{ deployment_name }}-postgres-configuration' namespace: '{{ meta.namespace }}' stringData: password: '{{ database_password }}' @@ -18,7 +18,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: '{{ tower_name }}-secret-key' + name: '{{ deployment_name }}-secret-key' namespace: '{{ meta.namespace }}' stringData: secret_key: '{{ secret_key }}' @@ -28,7 +28,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: '{{ tower_name }}-admin-password' + name: '{{ deployment_name }}-admin-password' namespace: '{{ meta.namespace }}' stringData: password: '{{ admin_password }}' @@ -38,7 +38,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: '{{ tower_name }}-broadcast-websocket' + name: '{{ deployment_name }}-broadcast-websocket' namespace: '{{ meta.namespace }}' stringData: secret: '{{ broadcast_websocket }}' diff --git a/roles/restore/vars/main.yml b/roles/restore/vars/main.yml index 851b98f0..c2a30003 100644 --- a/roles/restore/vars/main.yml +++ b/roles/restore/vars/main.yml @@ -1,4 +1,4 @@ --- deployment_type: "awx" -tower_postgres_image: postgres:12 +postgres_image: postgres:12 From b5c5a1722dc3d2d4dd1443b1fde88e0f55f81918 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 28 Apr 2021 01:24:05 -0400 Subject: [PATCH 38/43] revert unneccesary admin password update --- roles/restore/tasks/initialize_django.yml | 54 ----------------------- roles/restore/tasks/main.yml | 2 - 2 files changed, 56 deletions(-) delete mode 100644 roles/restore/tasks/initialize_django.yml diff --git a/roles/restore/tasks/initialize_django.yml b/roles/restore/tasks/initialize_django.yml deleted file mode 100644 index f1f217fe..00000000 --- a/roles/restore/tasks/initialize_django.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- - -- name: Get the new deployment resource pod information. - k8s_info: - api_version: v1 - kind: Pod - namespace: '{{ backup_pvc_namespace }}' - label_selectors: - - "app.kubernetes.io/name={{ deployment_name }}" - - "app.kubernetes.io/managed-by=awx-operator" - - "app.kubernetes.io/component=awx" - field_selectors: - - status.phase=Running - register: tower_pods - -- name: Set the resource pod name as a variable. - set_fact: - tower_pod_name: "{{ tower_pods['resources'][0]['metadata']['name'] | default('') }}" - -- name: Check if there are any super users defined. - k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ tower_pod_name }}" - container: "{{ meta.name }}-task" - command: >- - bash -c "echo 'from django.contrib.auth.models import User; - nsu = User.objects.filter(is_superuser=True, username='{{ tower_admin_user }}').count(); - exit(0 if nsu > 0 else 1)' - | awx-manage shell" - ignore_errors: true - register: users_result - changed_when: users_result.return_code > 0 - -- name: Update super user password via Django if it does exist (same password is a noop) - k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ tower_pod_name }}" - container: "{{ meta.name }}-task" - command: >- - bash -c "awx-manage update_password --username '{{ tower_admin_user }}' --password '{{ tower_admin_password }}'" - register: update_pw_result - changed_when: users_result.stdout == 'Password not updated' - when: users_result.return_code == 0 - -- name: Create super user via Django if it doesn't exist. - k8s_exec: - namespace: "{{ meta.namespace }}" - pod: "{{ tower_pod_name }}" - container: "{{ meta.name }}-task" - command: >- - bash -c "echo \"from django.contrib.auth.models import User; - User.objects.create_superuser('{{ tower_admin_user }}', '{{ tower_admin_email }}', '{{ tower_admin_password }}')\" - | awx-manage shell" - when: users_result.return_code > 0 diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index 5d861909..7fda2e1c 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -22,8 +22,6 @@ - include_tasks: postgres.yml - - include_tasks: initialize_django.yml - - name: Set flag signifying this restore was successful set_fact: tower_restore_complete: True From 36852cd5f5262132144c9373853dff7e701515c1 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 28 Apr 2021 01:43:19 -0400 Subject: [PATCH 39/43] remove unused variables in restore role --- roles/restore/defaults/main.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml index 387015d5..af7a06f5 100644 --- a/roles/restore/defaults/main.yml +++ b/roles/restore/defaults/main.yml @@ -9,9 +9,4 @@ backup_pvc_namespace: '' # Required: backup name, found on the awxbackup object backup_dir: '' -# TODO: Is this necessary? User's will be able to use the rekey role - -tower_admin_password_secret: "{{ deployment_name }}-admin-password" tower_postgres_configuration_secret: "{{ deployment_name }}-postgres-configuration" -tower_secret_key_secret: "{{ deployment_name }}-secret-key" -tower_broadcast_websocket_secret: "{{ deployment_name }}-broadcast-websocket" From 9cfb7921bc6b180e8529cf5400d5d088cb8f48be Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 28 Apr 2021 01:51:29 -0400 Subject: [PATCH 40/43] update templated files with new var names --- deploy/awx-operator.yaml | 34 ++++++++++++------------- deploy/crds/awxbackup_v1beta1_crd.yaml | 14 +++++----- deploy/crds/awxrestore_v1beta1_crd.yaml | 20 +++++++-------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 67cfe045..57fc0ea3 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -416,19 +416,19 @@ spec: spec: type: object properties: - tower_name: + deployment_name: description: Name of the deployment to be backed up type: string - tower_backup_pvc: + backup_pvc: description: Name of the PVC to be used for storing the backup type: string - tower_backup_pvc_namespace: + backup_pvc_namespace: description: Namespace PVC is in type: string - tower_backup_size: - description: Size of PVC + backup_storage_requirements: + description: Storage requirements for the PostgreSQL container type: string - tower_backup_storage_class: + backup_storage_class: description: Storage class to use when creating PVC for backup type: string tower_secret_key_secret: @@ -448,7 +448,7 @@ spec: type: string oneOf: - - required: ["tower_name"] + - required: ["deployment_name"] --- apiVersion: apiextensions.k8s.io/v1 @@ -478,20 +478,20 @@ spec: spec: type: object properties: - tower_name: + deployment_name: description: Name of the deployment to be restored to type: string - tower_backup_pvc: - description: Name of the PVC to be restored from, set as a status found on the awxbackup object (towerBackupClaim) + backup: + description: AWXBackup object name type: string - tower_backup_pvc_namespace: + backup_pvc: + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim) + type: string + backup_pvc_namespace: description: Namespace the PVC is in type: string - tower_backup_dir: - description: Backup directory name, set as a status found on the awxbackup object (towerBackupDirectory) - type: string - tower_backup: - description: AWXBackup object name + backup_dir: + description: Backup directory name, set as a status found on the awxbackup object (backupDirectory) type: string tower_secret_key_secret: description: Custom secret_key secret name @@ -509,7 +509,7 @@ spec: description: Label selector used to identify postgres pod for backing up data type: string oneOf: - - required: ["tower_name", "tower_backup_pvc_namespace"] + - required: ["deployment_name", "backup_pvc_namespace"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/crds/awxbackup_v1beta1_crd.yaml b/deploy/crds/awxbackup_v1beta1_crd.yaml index 21e388f3..fd5f04be 100644 --- a/deploy/crds/awxbackup_v1beta1_crd.yaml +++ b/deploy/crds/awxbackup_v1beta1_crd.yaml @@ -26,19 +26,19 @@ spec: spec: type: object properties: - tower_name: + deployment_name: description: Name of the deployment to be backed up type: string - tower_backup_pvc: + backup_pvc: description: Name of the PVC to be used for storing the backup type: string - tower_backup_pvc_namespace: + backup_pvc_namespace: description: Namespace PVC is in type: string - tower_backup_size: - description: Size of PVC + backup_storage_requirements: + description: Storage requirements for the PostgreSQL container type: string - tower_backup_storage_class: + backup_storage_class: description: Storage class to use when creating PVC for backup type: string tower_secret_key_secret: @@ -58,4 +58,4 @@ spec: type: string oneOf: - - required: ["tower_name"] + - required: ["deployment_name"] diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml index db5c4e13..8a91ce07 100644 --- a/deploy/crds/awxrestore_v1beta1_crd.yaml +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -26,20 +26,20 @@ spec: spec: type: object properties: - tower_name: + deployment_name: description: Name of the deployment to be restored to type: string - tower_backup_pvc: - description: Name of the PVC to be restored from, set as a status found on the awxbackup object (towerBackupClaim) + backup: + description: AWXBackup object name type: string - tower_backup_pvc_namespace: + backup_pvc: + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim) + type: string + backup_pvc_namespace: description: Namespace the PVC is in type: string - tower_backup_dir: - description: Backup directory name, set as a status found on the awxbackup object (towerBackupDirectory) - type: string - tower_backup: - description: AWXBackup object name + backup_dir: + description: Backup directory name, set as a status found on the awxbackup object (backupDirectory) type: string tower_secret_key_secret: description: Custom secret_key secret name @@ -57,4 +57,4 @@ spec: description: Label selector used to identify postgres pod for backing up data type: string oneOf: - - required: ["tower_name", "tower_backup_pvc_namespace"] + - required: ["deployment_name", "backup_pvc_namespace"] From 5439681a39b21887d10e008762e8b2243aae6dfe Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 28 Apr 2021 02:32:43 -0400 Subject: [PATCH 41/43] Fix rebase issue due to order or pg config tasks --- roles/installer/tasks/database_configuration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index 9c24b1fb..ef26b715 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -64,6 +64,10 @@ set_fact: pg_config: '{{ _generated_pg_config_resources["resources"] | default([]) | length | ternary(_generated_pg_config_resources, _pg_config) }}' +- name: Set actual postgres configuration secret used + set_fact: + postgres_configuration_secret: "{{ pg_config['resources'][0]['metadata']['name'] }}" + - block: - name: Create Database if no database is specified k8s: From cdbaf9460e1241adebaff26d015df769c49dd5ac Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Wed, 28 Apr 2021 13:52:45 -0400 Subject: [PATCH 42/43] Remove unnecessary intermediate awx_spec var --- roles/backup/tasks/awx-cro.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml index 3e487dd9..12693827 100644 --- a/roles/backup/tasks/awx-cro.yml +++ b/roles/backup/tasks/awx-cro.yml @@ -16,13 +16,9 @@ set_fact: awx_spec: "{{ _awx['spec'] }}" -- name: Template secrets into yaml - set_fact: - awx_definition_file: "{{ awx_spec }}" - - name: Write awx object to pvc k8s_exec: namespace: "{{ backup_pvc_namespace }}" pod: "{{ meta.name }}-db-management" command: >- - bash -c "echo '{{ awx_definition_file }}' > {{ backup_dir }}/awx_object" + bash -c "echo '{{ awx_spec }}' > {{ backup_dir }}/awx_object" From 5e2d11835e76145d9bcba6bbf91e1f2b075f00d9 Mon Sep 17 00:00:00 2001 From: "Christian M. Adams" Date: Fri, 30 Apr 2021 13:51:48 -0400 Subject: [PATCH 43/43] Fix rebase issue & remove dynamic kind/version var setting --- deploy/awx-operator.yaml | 1 + roles/backup/defaults/main.yml | 2 ++ roles/backup/tasks/error_handling.yml | 5 ----- roles/backup/tasks/init.yml | 7 +------ roles/backup/tasks/main.yml | 5 ----- roles/backup/tasks/update_status.yml | 4 ---- roles/installer/tasks/database_configuration.yml | 6 ------ roles/restore/defaults/main.yml | 2 ++ roles/restore/tasks/error_handling.yml | 5 ----- roles/restore/tasks/main.yml | 5 ----- roles/restore/tasks/update_status.yml | 4 ---- 11 files changed, 6 insertions(+), 40 deletions(-) diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 57fc0ea3..7677abe5 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -572,6 +572,7 @@ rules: - apps resources: - deployments/scale + - statefulsets/scale verbs: - patch - apiGroups: diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 484b54f2..b160118f 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -1,6 +1,8 @@ --- # Required: specify name of tower deployment to backup from deployment_name: '' +kind: 'AWXBackup' +api_version: '{{ deployment_type }}.ansible.com/v1beta1' # Specify a pre-created PVC (name) to backup to backup_pvc: '' diff --git a/roles/backup/tasks/error_handling.yml b/roles/backup/tasks/error_handling.yml index de1c28ce..1d41721d 100644 --- a/roles/backup/tasks/error_handling.yml +++ b/roles/backup/tasks/error_handling.yml @@ -1,10 +1,5 @@ --- -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - - name: Determine the timestamp set_fact: now: '{{ lookup("pipe", "date +%FT%TZ") }}' diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml index a0d208e6..42f593d9 100644 --- a/roles/backup/tasks/init.yml +++ b/roles/backup/tasks/init.yml @@ -60,15 +60,10 @@ template: "management-pod.yml.j2" wait: true -# Retrieve AWX object for use in postgres.yml and secrets.yml -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - - name: Look up details for this deployment k8s_info: api_version: "{{ api_version }}" - kind: "AWX" # Find a way to dynamically get this + kind: "AWX" name: "{{ deployment_name }}" namespace: "{{ meta.namespace }}" register: this_awx diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index f8203871..0ea4ce38 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -1,10 +1,5 @@ --- -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - - name: Look up details for this backup object k8s_info: api_version: "{{ api_version }}" diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml index fe55aa92..a497e86a 100644 --- a/roles/backup/tasks/update_status.yml +++ b/roles/backup/tasks/update_status.yml @@ -1,8 +1,4 @@ --- -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' # The backup directory in this status can be referenced when restoring - name: Update Tower Backup status diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index ef26b715..acf55554 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -113,12 +113,6 @@ awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] | default('prefer'|b64encode) | b64decode }}" - -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - - name: Look up details for this deployment k8s_info: api_version: "{{ api_version }}" diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml index af7a06f5..0a9422a9 100644 --- a/roles/restore/defaults/main.yml +++ b/roles/restore/defaults/main.yml @@ -1,6 +1,8 @@ --- # Required: specify name of tower deployment to restore to deployment_name: '' +kind: 'AWXRestore' +api_version: '{{ deployment_type }}.ansible.com/v1beta1' # Required: specify a pre-created PVC (name) to restore from backup_pvc: '' diff --git a/roles/restore/tasks/error_handling.yml b/roles/restore/tasks/error_handling.yml index de1c28ce..1d41721d 100644 --- a/roles/restore/tasks/error_handling.yml +++ b/roles/restore/tasks/error_handling.yml @@ -1,10 +1,5 @@ --- -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - - name: Determine the timestamp set_fact: now: '{{ lookup("pipe", "date +%FT%TZ") }}' diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml index 7fda2e1c..2710b2aa 100644 --- a/roles/restore/tasks/main.yml +++ b/roles/restore/tasks/main.yml @@ -1,10 +1,5 @@ --- -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - - name: Look up details for this restore object k8s_info: api_version: "{{ api_version }}" diff --git a/roles/restore/tasks/update_status.yml b/roles/restore/tasks/update_status.yml index 92196d01..2b63a884 100644 --- a/roles/restore/tasks/update_status.yml +++ b/roles/restore/tasks/update_status.yml @@ -1,8 +1,4 @@ --- -- name: Set apiVersion and kind variables - set_fact: - api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' - kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' - name: Update Tower Restore status operator_sdk.util.k8s_status: