init restore

This commit is contained in:
Christian M. Adams
2021-04-01 14:52:22 -04:00
parent 80c8d87f71
commit 8467209d35
31 changed files with 634 additions and 21 deletions

View File

@@ -18,6 +18,12 @@
dest: "{{ playbook_dir }}/../deploy/crds/awxbackup_v1beta1_crd.yaml"
mode: '0644'
- name: Template AWXRestore CRD
template:
src: awxrestore_crd.yml.j2
dest: "{{ playbook_dir }}/../deploy/crds/awxrestore_v1beta1_crd.yaml"
mode: '0644'
- name: Template awx-operator.yaml
template:
src: awx-operator.yaml.j2

View File

@@ -14,3 +14,5 @@
{% include 'crd.yml.j2' %}
{% include 'awxbackup_crd.yml.j2' %}
{% include 'awxbackup_crd.yml.j2' %}

View File

@@ -0,0 +1,51 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: awxrestores.awx.ansible.com
spec:
group: awx.ansible.com
names:
kind: AWXRestore
listKind: AWXRestoreList
plural: awxrestores
singular: awxrestore
scope: Namespaced
versions:
- name: v1beta1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
description: Schema validation for the AWXBackup CRD
properties:
spec:
type: object
properties:
tower_name:
description: Name of the deployment to be restored to
type: string
tower_backup_pvc:
description: Name of the PVC to be used for storing the backup
type: string
tower_backup_dir:
description: Backup directory name, a status found on the awxbackup object (towerBackupComplete)
type: string
tower_secret_key_secret:
description: Custom secret_key secret name
type: string
tower_admin_password_secret:
description: Custom admin_password secret name
type: string
tower_broadcast_websocket_secret:
description: Custom broadcast_websocket secret name
type: string
tower_postgres_configuration_secret:
description: Custom postgres_configuration secret name
type: string
oneOf:
- required: ["tower_name", "tower_backup_pvc"]

View File

@@ -580,3 +580,58 @@ spec:
type: string
oneOf:
- required: ["tower_name"]
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: awxbackups.awx.ansible.com
spec:
group: awx.ansible.com
names:
kind: AWXBackup
listKind: AWXBackupList
plural: awxbackups
singular: awxbackup
scope: Namespaced
versions:
- name: v1beta1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
description: Schema validation for the AWXBackup CRD
properties:
spec:
type: object
properties:
tower_name:
description: Name of the deployment to be backed up
type: string
tower_backup_pvc:
description: Name of the PVC to be used for storing the backup
type: string
tower_backup_size:
description: Size of PVC
type: string
tower_backup_storage_class:
description: Storage class to use when creating PVC for backup
type: string
tower_secret_key_secret:
description: Custom secret_key secret name
type: string
tower_admin_password_secret:
description: Custom admin_password secret name
type: string
tower_broadcast_websocket_secret:
description: Custom broadcast_websocket secret name
type: string
tower_postgres_configuration_secret:
description: Custom postgres_configuration secret name
type: string
oneOf:
- required: ["tower_name"]

View File

@@ -1,12 +0,0 @@
---
apiVersion: awx.ansible.com/v1beta1
kind: AWXBackup
metadata:
name: example-awxbackup
namespace: example-awx
spec:
tower_name: ''
tower_backup_pvc: ''
tower_backup_size: ''
tower_backup_storage_class: ''
tower_postgres_configuration_secret: ''

View File

@@ -0,0 +1,51 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: awxrestores.awx.ansible.com
spec:
group: awx.ansible.com
names:
kind: AWXRestore
listKind: AWXRestoreList
plural: awxrestores
singular: awxrestore
scope: Namespaced
versions:
- name: v1beta1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
description: Schema validation for the AWXBackup CRD
properties:
spec:
type: object
properties:
tower_name:
description: Name of the deployment to be restored to
type: string
tower_backup_pvc:
description: Name of the PVC to be used for storing the backup
type: string
tower_backup_dir:
description: Backup directory name, a status found on the awxbackup object (towerBackupComplete)
type: string
tower_secret_key_secret:
description: Custom secret_key secret name
type: string
tower_admin_password_secret:
description: Custom admin_password secret name
type: string
tower_broadcast_websocket_secret:
description: Custom broadcast_websocket secret name
type: string
tower_postgres_configuration_secret:
description: Custom postgres_configuration secret name
type: string
oneOf:
- required: ["tower_name", "tower_backup_pvc"]

View File

@@ -19,6 +19,10 @@
k8s:
definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxbackup_v1beta1_crd.yaml'])) }}"
- name: Create AWXBackup Custom Resource Definition
k8s:
definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxrestore_v1beta1_crd.yaml'])) }}"
- name: Ensure specified namespace is present
k8s:
api_version: v1

View File

@@ -83,7 +83,7 @@
set_fact:
broadcast_websocket_template: "{{ lookup('file', '_secrets/broadcast_websocket_secret.yml') }}"
- name: Write secret_key to pvc
- name: Write broadcast_websocket definition to pvc
community.kubernetes.k8s_exec:
namespace: "{{ meta.namespace }}"
pod: "{{ meta.name }}-db-management"
@@ -116,7 +116,7 @@
set_fact:
postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yml') }}"
- name: Write secret_key to pvc
- name: Write postgres configuration to pvc
community.kubernetes.k8s_exec:
namespace: "{{ meta.namespace }}"
pod: "{{ meta.name }}-db-management"

View File

@@ -3,7 +3,7 @@ apiVersion: v1
kind: Secret
metadata:
{% raw %}
name: '{{ meta.name }}'
name: '{{ tower_name }}'
namespace: '{{ meta.namespace }}'
{% endraw %}
stringData:

View File

@@ -3,7 +3,7 @@ apiVersion: '{{ awx_api_version }}'
kind: AWX
metadata:
{% raw %}
name: '{{ meta.name }}'
name: '{{ tower_name }}'
namespace: '{{ meta.namespace }}'
{% endraw %}
spec: {{ awx_spec }}

View File

@@ -3,7 +3,7 @@ apiVersion: v1
kind: Secret
metadata:
{% raw %}
name: '{{ meta.name }}-broadcast-websocket'
name: '{{ tower_name }}-broadcast-websocket'
namespace: '{{ meta.namespace }}'
{% endraw %}
stringData:

View File

@@ -2,7 +2,7 @@
apiVersion: v1
kind: Event
metadata:
name: backup-error.{{ now }}
name: restore-error.{{ now }}
namespace: {{ meta.namespace }}
involvedObject:
apiVersion: awx.ansible.com/v1beta1
@@ -10,7 +10,7 @@ involvedObject:
name: {{ meta.name }}
namespace: {{ meta.namespace }}
message: {{ error_msg }}
reason: BackupFailed
reason: RestoreFailed
type: Warning
firstTimestamp: {{ now }}
lastTimestamp: {{ now }}

View File

@@ -4,7 +4,7 @@ apiVersion: v1
kind: Secret
metadata:
{% raw %}
name: '{{ meta.name }}-postgres-configuration'
name: '{{ tower_name }}-postgres-configuration'
namespace: '{{ meta.namespace }}'
{% endraw %}
stringData:

View File

@@ -3,7 +3,7 @@ apiVersion: v1
kind: Secret
metadata:
{% raw %}
name: '{{ meta.name }}'
name: '{{ tower_name }}'
namespace: '{{ meta.namespace }}'
{% endraw %}
stringData:

Binary file not shown.

82
roles/restore/README.md Normal file
View File

@@ -0,0 +1,82 @@
Role Name
=========
The purpose of this role is to restore your AWX deployment from an existing PVC backup. The backup should include:
- backup of the postgresql database
- secrets, included the secret_key.
- AWX custom resource object with deployment specific settings
Requirements
------------
This role assumes you are authenticated with an Openshift or Kubernetes cluster which:
- The awx-operator has been deployed to
- AWX is deployed to via the operator
Usage
----------------
Then create a file named `restore-awx.yml` with the following contents:
```yaml
---
apiVersion: awx.ansible.com/v1beta1
kind: AWXRestore
metadata:
name: awxrestore
namespace: my-namespace
spec:
tower_name: mytower
tower_backup_pvc: myoldtower-awxbackup-adfx7ciow
tower_backup_dir: tower-openshift-backup-2021-04-01-15:49:17
```
Note that the `tower_name` above is the name of the AWX deployment you intend to create and restore to.
Finally, use `kubectl` to create the restore object in your cluster:
```bash
#> kubectl apply -f restore-awx.yml
```
This will create a new deployment and restore your backup to it.
Role Variables
--------------
A custom, pre-created pvc can be used by setting the following variables.
```
tower_backup_pvc: 'awx-backup-volume-claim'
```
If a custom postgres configuration secret was used when deploying AWX, it must be set:
```
tower_postgres_configuration_secret: 'awx-postgres-configuration'
```
Testing
----------------
You can test this role directly by creating and running the following playbook with the appropriate variables:
```
---
- name: Backup Tower
hosts: localhost
gather_facts: false
roles:
- backup
```
License
-------
MIT

View File

@@ -0,0 +1,17 @@
---
# Required: specify name of tower deployment to restore to
tower_name: ''
# Required: specify a pre-created PVC (name) to restore from
tower_backup_pvc: ''
# Required: backup name, found on the awxbackup object
tower_backup_dir: ''
# TODO: Should we add a unique id at the end of the secret when backing up, then use it here?
# or will that make future backups more complicated because the user will have to specify the names of all the secrets?
# Names of any secrets you want to use instead of the ones in the backup
tower_secret_key_secret: "{{ tower_name }}-secret-key"
tower_admin_password_secret: "{{ tower_name }}-admin-password"
tower_broadcast_websocket_secret: "{{ tower_name }}-broadcast-websocket"
tower_postgres_configuration_secret: "{{ tower_name }}-postgres-configuration"

View File

@@ -0,0 +1,30 @@
---
galaxy_info:
author: Ansible
description: AWX role for AWX Operator for Kubernetes.
company: Red Hat, Inc.
license: MIT
min_ansible_version: 2.8
platforms:
- name: EL
versions:
- all
- name: Debian
versions:
- all
galaxy_tags:
- tower
- awx
- ansible
- restore
- automation
dependencies: []
collections:
- community.kubernetes
- operator_sdk.util

View File

@@ -0,0 +1,9 @@
---
- name: Delete any existing management pod
community.kubernetes.k8s:
name: "{{ meta.name }}-db-management"
kind: Pod
namespace: "{{ meta.namespace }}"
state: absent
force: true

View File

@@ -0,0 +1,16 @@
---
- name: Set apiVersion and kind variables
set_fact:
api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}'
kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}'
- name: Determine the timestamp
set_fact:
now: '{{ lookup("pipe", "date +%FT%TZ") }}'
- name: Emit ocp event with error
community.kubernetes.k8s:
kind: Event
namespace: "{{ meta.namespace }}"
template: "event.yml.j2"

View File

@@ -0,0 +1,55 @@
---
- name: Delete any existing management pod
community.kubernetes.k8s:
name: "{{ meta.name }}-db-management"
kind: Pod
namespace: "{{ meta.namespace }}"
state: absent
force: true
wait: true
# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever.
- name: Check provided PVC exists
k8s_info:
name: "{{ tower_backup_pvc }}"
kind: PersistentVolumeClaim
namespace: "{{ meta.namespace }}"
register: provided_pvc
when:
- tower_backup_pvc != ''
- name: Surface error to user
block:
- name: Set error message
set_fact:
error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first."
- name: Handle error
import_tasks: error_handling.yml
- name: Fail early if pvc is defined but does not exist
fail:
msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first."
when:
- tower_backup_pvc != ''
- provided_pvc.resources | length == 0
# If tower_backup_pvc is defined, use in management-pod.yml.j2
- name: Set default pvc name
set_fact:
_default_backup_pvc: "{{ meta.name }}-backup-claim"
# by default, it will re-use the old pvc if already created (unless pvc is provided)
- name: Set PVC to use for backup
set_fact:
backup_pvc: "{{ tower_backup_pvc | default(_default_backup_pvc, true) }}"
- name: Create management pod from templated deployment config
community.kubernetes.k8s:
name: "{{ meta.name }}-db-management"
kind: Deployment
namespace: "{{ meta.namespace }}"
state: present
template: "management-pod.yml.j2"
wait: true

View File

@@ -0,0 +1,12 @@
---
- name: Deploy AWX
k8s:
state: "{{ state | default('present') }}"
namespace: "{{ tower_namespace | default('default') }}"
apply: yes
wait: yes
template: "{{ tower_backup_dir }}/awx_object.yml"
# TODO: Add logic to allow users to provide override values here,
# or to specify spec values that were not in the backed up AWX object.
# This may involve changing how we back up the spec section of the AWX object

View File

@@ -0,0 +1,39 @@
---
- name: Set apiVersion and kind variables
set_fact:
api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}'
kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}'
- name: Look up details for this deployment
k8s_info:
api_version: "{{ api_version }}"
kind: "{{ kind }}"
name: "{{ meta.name }}"
namespace: "{{ meta.namespace }}"
register: this_restore
- block:
- include_tasks: preflight.yml
- include_tasks: init_awx.yml
- include_tasks: init.yml
# - include_tasks: postgres.yml
#
# - include_tasks: secrets.yml
- name: Set flag signifying this backup was successful
set_fact:
tower_backup_complete: True
- include_tasks: cleanup.yml
when:
- this_restore['resources'][0]['status']['towerRestoreComplete'] is not defined
- name: Update status variables
include_tasks: update_status.yml
# TODO: backup tower settings or make sure that users only specify settigns/config changes via AWX object. See ticket

View File

@@ -0,0 +1,85 @@
---
- name: Check for specified PostgreSQL configuration
k8s_info:
kind: Secret
namespace: '{{ meta.namespace }}'
name: '{{ tower_postgres_configuration_secret }}'
register: _custom_pg_config_resources
when: tower_postgres_configuration_secret | length
- name: Check for default PostgreSQL configuration
k8s_info:
kind: Secret
namespace: '{{ meta.namespace }}'
name: '{{ tower_name }}-postgres-configuration'
register: _default_pg_config_resources
- name: Set PostgreSQL configuration
set_fact:
pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}'
- name: Store Database Configuration
set_fact:
awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}"
awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}"
awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
- name: Get the postgres pod information
k8s_info:
kind: Pod
namespace: '{{ meta.namespace }}'
label_selectors:
- "app={{ tower_name }}-{{ deployment_type }}-postgres"
register: postgres_pod
until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
delay: 5
retries: 60
- name: Set the resource pod name as a variable.
set_fact:
postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
# TODO: Add postgres restore logic, just pipe the pg_dump from PVC to db via psql
#
# - name: Create directory for backup
# community.kubernetes.k8s_exec:
# namespace: "{{ meta.namespace }}"
# pod: "{{ meta.name }}-db-management"
# command: >-
# mkdir -p {{ _backup_dir }}
#
# - name: Precreate file for database dump
# community.kubernetes.k8s_exec:
# namespace: "{{ meta.namespace }}"
# pod: "{{ meta.name }}-db-management"
# command: >-
# touch {{ _backup_dir }}/tower.db
#
# - name: Set permissions on file for database dump
# community.kubernetes.k8s_exec:
# namespace: "{{ meta.namespace }}"
# pod: "{{ meta.name }}-db-management"
# command: >-
# chmod 0600 {{ _backup_dir }}/tower.db
#
# - name: Set pg_dump command
# set_fact:
# pgdump: >-
# pg_dump --clean --create
# -h {{ awx_postgres_host }}
# -U {{ awx_postgres_user }}
# -d {{ awx_postgres_database }}
# -p {{ awx_postgres_port }}
#
# - name: Write pg_dump to backup on PVC
# community.kubernetes.k8s_exec:
# namespace: "{{ meta.namespace }}"
# pod: "{{ meta.name }}-db-management"
# command: >-
# bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/tower.db"
# register: data_migration

View File

@@ -0,0 +1,27 @@
---
# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever.
- name: Check provided PVC exists
k8s_info:
name: "{{ tower_backup_pvc }}"
kind: PersistentVolumeClaim
namespace: "{{ meta.namespace }}"
register: provided_pvc
when:
- tower_backup_pvc != ''
- name: Surface error to user
block:
- name: Set error message
set_fact:
error_msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first."
- name: Handle error
import_tasks: error_handling.yml
- name: Fail early if pvc is defined but does not exist
fail:
msg: "{{ tower_backup_pvc }} does not exist, please create this pvc first."
when:
- tower_backup_pvc != ''
- provided_pvc.resources | length == 0

View File

@@ -0,0 +1,21 @@
---
- name: Create secret_key secret
k8s:
definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'secret_key_secret.yml'])) }}"
- name: Create admin_password secret
k8s:
definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'admin_password_secret.yml'])) }}"
- name: Create broadcast_websocket secret
k8s:
definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'broadcast_websocket_secret.yml'])) }}"
- name: Create postgres configuration secret
k8s:
definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'postgres_secret.yml'])) }}"
- name: Create secret_key secret
k8s:
definition: "{{ lookup('file', '/'.join([tower_backup_dir, 'secret_key_secret.yml'])) }}"

View File

@@ -0,0 +1,15 @@
---
- name: Set apiVersion and kind variables
set_fact:
api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}'
kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}'
- name: Update Tower Backup status
operator_sdk.util.k8s_status:
api_version: '{{ api_version }}'
kind: "{{ kind }}"
name: "{{ meta.name }}"
namespace: "{{ meta.namespace }}"
status:
towerRestoreComplete: True
when: tower_restore_complete is defined

View File

@@ -0,0 +1,17 @@
---
apiVersion: v1
kind: Event
metadata:
name: backup-error.{{ now }}
namespace: {{ meta.namespace }}
involvedObject:
apiVersion: awx.ansible.com/v1beta1
kind: {{ kind }}
name: {{ meta.name }}
namespace: {{ meta.namespace }}
message: {{ error_msg }}
reason: BackupFailed
type: Warning
firstTimestamp: {{ now }}
lastTimestamp: {{ now }}
count: 1

View File

@@ -0,0 +1,22 @@
---
apiVersion: v1
kind: Pod
metadata:
name: {{ meta.name }}-db-management
namespace: {{ meta.namespace }}
spec:
containers:
- name: {{ meta.name }}-db-management
image: "{{ tower_postgres_image }}"
imagePullPolicy: Always
command: ["sleep", "infinity"]
volumeMounts:
- name: {{ meta.name }}-backup
mountPath: /backups
readOnly: false
volumes:
- name: {{ meta.name }}-backup
persistentVolumeClaim:
claimName: {{ backup_pvc }}
readOnly: false
restartPolicy: Never

View File

@@ -0,0 +1,4 @@
---
deployment_type: "awx"
tower_postgres_image: postgres:12

View File

@@ -11,3 +11,8 @@
group: awx.ansible.com
kind: AWXBackup
role: backup
- version: v1beta1
group: awx.ansible.com
kind: AWXRestore
role: restore