diff --git a/README.md b/README.md index bd98aae8..ce009bc0 100644 --- a/README.md +++ b/README.md @@ -229,9 +229,12 @@ stringData: username: password: sslmode: prefer + type: unmanaged type: Opaque ``` +> It is possible to set a specific username, password, port, or database, but still have the database managed by the operator. In this case, when creating the postgres-configuration secret, the `type: managed` field should be added. + **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`. #### Migrating data from an old AWX instance @@ -665,7 +668,7 @@ After it is built, test it on a local cluster: #> minikube addons enable ingress #> ansible-playbook ansible/deploy-operator.yml -e operator_image=quay.io//awx-operator -e operator_version=test #> kubectl create namespace example-awx -#> ansible-playbook ansible/instantiate-awx-deployment.yml -e tower_namespace=example-awx +#> ansible-playbook ansible/instantiate-awx-deployment.yml -e namespace=example-awx #> #> minikube delete ``` diff --git a/ansible/chain-operator-files.yml b/ansible/chain-operator-files.yml index 03f460bf..23cfc05a 100644 --- a/ansible/chain-operator-files.yml +++ b/ansible/chain-operator-files.yml @@ -6,12 +6,24 @@ gather_facts: false tasks: - - name: Template CRD + - name: Template AWX CRD template: src: crd.yml.j2 dest: "{{ playbook_dir }}/../deploy/crds/awx_v1beta1_crd.yaml" mode: '0644' + - name: Template AWXBackup CRD + template: + src: awxbackup_crd.yml.j2 + dest: "{{ playbook_dir }}/../deploy/crds/awxbackup_v1beta1_crd.yaml" + mode: '0644' + + - name: Template AWXRestore CRD + template: + src: awxrestore_crd.yml.j2 + dest: "{{ playbook_dir }}/../deploy/crds/awxrestore_v1beta1_crd.yaml" + mode: '0644' + - name: Template awx-operator.yaml template: src: awx-operator.yaml.j2 diff --git a/ansible/instantiate-awx-deployment.yml b/ansible/instantiate-awx-deployment.yml index f5195185..3a0c4175 100644 --- a/ansible/instantiate-awx-deployment.yml +++ b/ansible/instantiate-awx-deployment.yml @@ -9,7 +9,7 @@ - name: Deploy AWX k8s: state: "{{ state | default('present') }}" - namespace: "{{ tower_namespace | default('default') }}" + namespace: "{{ namespace | default('default') }}" apply: yes wait: yes definition: diff --git a/ansible/templates/awx-operator.yaml.j2 b/ansible/templates/awx-operator.yaml.j2 index 6d1364ba..21a74d67 100644 --- a/ansible/templates/awx-operator.yaml.j2 +++ b/ansible/templates/awx-operator.yaml.j2 @@ -3,6 +3,10 @@ # Update templates under ansible/templates/ {% include 'crd.yml.j2' %} +{% include 'awxbackup_crd.yml.j2' %} + +{% include 'awxrestore_crd.yml.j2' %} + {% include 'role.yml.j2' %} {% include 'role_binding.yml.j2' %} diff --git a/ansible/templates/awxbackup_crd.yml.j2 b/ansible/templates/awxbackup_crd.yml.j2 new file mode 100644 index 00000000..fd5f04be --- /dev/null +++ b/ansible/templates/awxbackup_crd.yml.j2 @@ -0,0 +1,61 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxbackups.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXBackup + listKind: AWXBackupList + plural: awxbackups + singular: awxbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + properties: + spec: + type: object + properties: + deployment_name: + description: Name of the deployment to be backed up + type: string + backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + backup_pvc_namespace: + description: Namespace PVC is in + type: string + backup_storage_requirements: + description: Storage requirements for the PostgreSQL container + type: string + backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + + oneOf: + - required: ["deployment_name"] diff --git a/ansible/templates/awxrestore_crd.yml.j2 b/ansible/templates/awxrestore_crd.yml.j2 new file mode 100644 index 00000000..8a91ce07 --- /dev/null +++ b/ansible/templates/awxrestore_crd.yml.j2 @@ -0,0 +1,60 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxrestores.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXRestore + listKind: AWXRestoreList + plural: awxrestores + singular: awxrestore + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXRestore CRD + properties: + spec: + type: object + properties: + deployment_name: + description: Name of the deployment to be restored to + type: string + backup: + description: AWXBackup object name + type: string + backup_pvc: + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim) + type: string + backup_pvc_namespace: + description: Namespace the PVC is in + type: string + backup_dir: + description: Backup directory name, set as a status found on the awxbackup object (backupDirectory) + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + oneOf: + - required: ["deployment_name", "backup_pvc_namespace"] diff --git a/ansible/templates/crd.yml.j2 b/ansible/templates/crd.yml.j2 index ad8ca846..0bca0a1c 100644 --- a/ansible/templates/crd.yml.j2 +++ b/ansible/templates/crd.yml.j2 @@ -58,6 +58,9 @@ spec: tower_old_postgres_configuration_secret: description: Secret where the old database configuration can be found for data migration type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string @@ -346,7 +349,16 @@ spec: description: Admin user of the deployed instance type: string towerAdminPasswordSecret: - description: Admin password of the deployed instance + description: Admin password secret name of the deployed instance + type: string + towerPostgresConfigurationSecret: + description: Postgres Configuration secret name of the deployed instance + type: string + towerBroadcastWebsocketSecret: + description: Broadcast websocket secret name of the deployed instance + type: string + towerSecretKeySecret: + description: Secret key secret name of the deployed instance type: string towerMigratedFromSecret: description: The secret used for migrating an old Tower. diff --git a/ansible/templates/role.yml.j2 b/ansible/templates/role.yml.j2 index 1b1263b3..886bd5f0 100644 --- a/ansible/templates/role.yml.j2 +++ b/ansible/templates/role.yml.j2 @@ -79,5 +79,7 @@ rules: - awx.ansible.com resources: - '*' + - awxbackups + - awxrestores verbs: - '*' diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 7c1f3636..00ca1c23 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -60,6 +60,9 @@ spec: tower_old_postgres_configuration_secret: description: Secret where the old database configuration can be found for data migration type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string @@ -348,7 +351,16 @@ spec: description: Admin user of the deployed instance type: string towerAdminPasswordSecret: - description: Admin password of the deployed instance + description: Admin password secret name of the deployed instance + type: string + towerPostgresConfigurationSecret: + description: Postgres Configuration secret name of the deployed instance + type: string + towerBroadcastWebsocketSecret: + description: Broadcast websocket secret name of the deployed instance + type: string + towerSecretKeySecret: + description: Secret key secret name of the deployed instance type: string towerMigratedFromSecret: description: The secret used for migrating an old Tower. @@ -376,6 +388,129 @@ spec: type: object type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxbackups.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXBackup + listKind: AWXBackupList + plural: awxbackups + singular: awxbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + properties: + spec: + type: object + properties: + deployment_name: + description: Name of the deployment to be backed up + type: string + backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + backup_pvc_namespace: + description: Namespace PVC is in + type: string + backup_storage_requirements: + description: Storage requirements for the PostgreSQL container + type: string + backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + + oneOf: + - required: ["deployment_name"] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxrestores.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXRestore + listKind: AWXRestoreList + plural: awxrestores + singular: awxrestore + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXRestore CRD + properties: + spec: + type: object + properties: + deployment_name: + description: Name of the deployment to be restored to + type: string + backup: + description: AWXBackup object name + type: string + backup_pvc: + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim) + type: string + backup_pvc_namespace: + description: Namespace the PVC is in + type: string + backup_dir: + description: Backup directory name, set as a status found on the awxbackup object (backupDirectory) + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + oneOf: + - required: ["deployment_name", "backup_pvc_namespace"] + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -457,6 +592,8 @@ rules: - awx.ansible.com resources: - '*' + - awxbackups + - awxrestores verbs: - '*' diff --git a/deploy/crds/awx_v1beta1_crd.yaml b/deploy/crds/awx_v1beta1_crd.yaml index ad8ca846..0bca0a1c 100644 --- a/deploy/crds/awx_v1beta1_crd.yaml +++ b/deploy/crds/awx_v1beta1_crd.yaml @@ -58,6 +58,9 @@ spec: tower_old_postgres_configuration_secret: description: Secret where the old database configuration can be found for data migration type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for data migration + type: string tower_secret_key_secret: description: Secret where the secret key can be found type: string @@ -346,7 +349,16 @@ spec: description: Admin user of the deployed instance type: string towerAdminPasswordSecret: - description: Admin password of the deployed instance + description: Admin password secret name of the deployed instance + type: string + towerPostgresConfigurationSecret: + description: Postgres Configuration secret name of the deployed instance + type: string + towerBroadcastWebsocketSecret: + description: Broadcast websocket secret name of the deployed instance + type: string + towerSecretKeySecret: + description: Secret key secret name of the deployed instance type: string towerMigratedFromSecret: description: The secret used for migrating an old Tower. diff --git a/deploy/crds/awxbackup_v1beta1_crd.yaml b/deploy/crds/awxbackup_v1beta1_crd.yaml new file mode 100644 index 00000000..fd5f04be --- /dev/null +++ b/deploy/crds/awxbackup_v1beta1_crd.yaml @@ -0,0 +1,61 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxbackups.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXBackup + listKind: AWXBackupList + plural: awxbackups + singular: awxbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXBackup CRD + properties: + spec: + type: object + properties: + deployment_name: + description: Name of the deployment to be backed up + type: string + backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + backup_pvc_namespace: + description: Namespace PVC is in + type: string + backup_storage_requirements: + description: Storage requirements for the PostgreSQL container + type: string + backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + + oneOf: + - required: ["deployment_name"] diff --git a/deploy/crds/awxrestore_v1beta1_crd.yaml b/deploy/crds/awxrestore_v1beta1_crd.yaml new file mode 100644 index 00000000..8a91ce07 --- /dev/null +++ b/deploy/crds/awxrestore_v1beta1_crd.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: awxrestores.awx.ansible.com +spec: + group: awx.ansible.com + names: + kind: AWXRestore + listKind: AWXRestoreList + plural: awxrestores + singular: awxrestore + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the AWXRestore CRD + properties: + spec: + type: object + properties: + deployment_name: + description: Name of the deployment to be restored to + type: string + backup: + description: AWXBackup object name + type: string + backup_pvc: + description: Name of the PVC to be restored from, set as a status found on the awxbackup object (backupClaim) + type: string + backup_pvc_namespace: + description: Namespace the PVC is in + type: string + backup_dir: + description: Backup directory name, set as a status found on the awxbackup object (backupDirectory) + type: string + tower_secret_key_secret: + description: Custom secret_key secret name + type: string + tower_admin_password_secret: + description: Custom admin_password secret name + type: string + tower_broadcast_websocket_secret: + description: Custom broadcast_websocket secret name + type: string + tower_postgres_configuration_secret: + description: Custom postgres_configuration secret name + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for backing up data + type: string + oneOf: + - required: ["deployment_name", "backup_pvc_namespace"] diff --git a/docs/migration.md b/docs/migration.md index f2b258d8..e83397e1 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -6,14 +6,14 @@ To migrate data from an older AWX installation, you must provide some informatio ### Secret Key -You can find your old secret key in the inventory file you used to deploy AWX in releases prior to version 18. +You can find your old secret key in the inventory file you used to deploy AWX in releases prior to version 18. ```yaml apiVersion: v1 kind: Secret metadata: name: -secret-key - namespace: + namespace: stringData: secret_key: type: Opaque @@ -49,6 +49,9 @@ In the next section pass it in through `tower_postgres_configuration_secret` ins from the key and ensuring the value matches the name of the secret. This will make AWX pick up on the existing database and apply any pending migrations. It is strongly recommended to backup your database beforehand. +The postgresql pod for the old deployment is used when streaming data to the new postgresql pod. If your postgresql pod has a custom label, +you can pass that via the `postgres_label_selector` variable to make sure the postgresql pod can be found. + ## Deploy AWX When you apply your AWX object, you must specify the name to the database secret you created above: diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index a5edb8fb..118388ea 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -11,10 +11,18 @@ - "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/ansible/group_vars/all" tasks: - - name: Create Custom Resource Definition + - name: Create AWX Custom Resource Definition k8s: definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_crd.yaml'])) }}" + - name: Create AWXBackup Custom Resource Definition + k8s: + definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxbackup_v1beta1_crd.yaml'])) }}" + + - name: Create AWXRestore Custom Resource Definition + k8s: + definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awxrestore_v1beta1_crd.yaml'])) }}" + - name: Ensure specified namespace is present k8s: api_version: v1 diff --git a/molecule/test-local/converge.yml b/molecule/test-local/converge.yml index 58790fe0..ee5f1a35 100644 --- a/molecule/test-local/converge.yml +++ b/molecule/test-local/converge.yml @@ -29,10 +29,9 @@ operator_image: awx.ansible.com/awx-operator operator_version: testing custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_molecule.yaml'])) | from_yaml }}" + tasks: - - block: - - name: Delete the Operator Deployment k8s: state: absent diff --git a/roles/backup/README.md b/roles/backup/README.md new file mode 100644 index 00000000..3f482f72 --- /dev/null +++ b/roles/backup/README.md @@ -0,0 +1,95 @@ +Backup Role +========= + +The purpose of this role is to create a backup of your AWX deployment which includes: + - custom deployment specific values in the spec section of the AWX custom resource object + - backup of the postgresql database + - secret_key, admin_password, and broadcast_websocket secrets + - database configuration + +Requirements +------------ + +This role assumes you are authenticated with an Openshift or Kubernetes cluster: + - The awx-operator has been deployed to the cluster + - AWX is deployed to via the operator + + +Usage +---------------- + +Then create a file named `backup-awx.yml` with the following contents: + +```yaml +--- +apiVersion: awx.ansible.com/v1beta1 +kind: AWXBackup +metadata: + name: awxbackup-2021-04-22 + namespace: my-namespace +spec: + deployment_name: mytower +``` + +Note that the `deployment_name` above is the name of the AWX deployment you intend to backup from. The namespace above is the one containing the AWX deployment that will be backed up. + +Finally, use `kubectl` to create the backup object in your cluster: + +```bash +$ kubectl apply -f backup-awx.yml +``` + +The resulting pvc will contain a backup tar that can be used to restore to a new deployment. Future backups will also be stored in separate tars on the same pvc. + + +Role Variables +-------------- + +A custom, pre-created pvc can be used by setting the following variables. + +``` +backup_pvc: 'awx-backup-volume-claim' +``` + +> If no pvc or storage class is provided, the cluster's default storage class will be used to create the pvc. + +This role will automatically create a pvc using a Storage Class if provided: + +``` +backup_storage_class: 'standard' +backup_storage_requirements: '20Gi' +``` + +By default, the backup pvc will be created in the same namespace the awxbackup object is created in. If you want your backup to be stored +in a specific namespace, you can do so by specifying `backup_pvc_namespace`. Keep in mind that you will +need to provide the same namespace when restoring. + +``` +backup_pvc_namespace: 'custom-namespace' +``` + +If a custom postgres configuration secret was used when deploying AWX, it will automatically be used by the backup role. +To check the name of this secret, look at the towerPostgresConfigurationSecret status on your AWX object. + +The postgresql pod for the old deployment is used when backing up data to the new postgresql pod. If your postgresql pod has a custom label, +you can pass that via the `postgres_label_selector` variable to make sure the postgresql pod can be found. + + +Testing +---------------- + +You can test this role directly by creating and running the following playbook with the appropriate variables: + +``` +--- +- name: Backup AWX + hosts: localhost + gather_facts: false + roles: + - backup +``` + +License +------- + +MIT diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml new file mode 100644 index 00000000..b160118f --- /dev/null +++ b/roles/backup/defaults/main.yml @@ -0,0 +1,15 @@ +--- +# Required: specify name of tower deployment to backup from +deployment_name: '' +kind: 'AWXBackup' +api_version: '{{ deployment_type }}.ansible.com/v1beta1' + +# Specify a pre-created PVC (name) to backup to +backup_pvc: '' +backup_pvc_namespace: "{{ meta.namespace }}" + +# Size of backup PVC if created dynamically +backup_storage_requirements: '' + +# Specify storage class to determine how to dynamically create PVC's with +backup_storage_class: '' diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml new file mode 100644 index 00000000..3d3cd361 --- /dev/null +++ b/roles/backup/meta/main.yml @@ -0,0 +1,31 @@ +--- +galaxy_info: + author: Ansible + description: AWX role for AWX Operator for Kubernetes. + company: Red Hat, Inc. + + license: MIT + + min_ansible_version: 2.8 + + platforms: + - name: EL + versions: + - all + - name: Debian + versions: + - all + + galaxy_tags: + - tower + - controller + - awx + - ansible + - backup + - automation + +dependencies: [] + +collections: + - community.kubernetes + - operator_sdk.util diff --git a/roles/backup/tasks/awx-cro.yml b/roles/backup/tasks/awx-cro.yml new file mode 100644 index 00000000..12693827 --- /dev/null +++ b/roles/backup/tasks/awx-cro.yml @@ -0,0 +1,24 @@ +--- + +- name: Get AWX custom resource object + k8s_info: + version: v1beta1 + kind: AWX + namespace: '{{ meta.namespace }}' + name: '{{ deployment_name }}' + register: _awx_cro + +- name: Set AWX object + set_fact: + _awx: "{{ _awx_cro['resources'][0] }}" + +- name: Set user specified spec + set_fact: + awx_spec: "{{ _awx['spec'] }}" + +- name: Write awx object to pvc + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ awx_spec }}' > {{ backup_dir }}/awx_object" diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml new file mode 100644 index 00000000..f91c8e6c --- /dev/null +++ b/roles/backup/tasks/cleanup.yml @@ -0,0 +1,9 @@ +--- + +- name: Delete any existing management pod + k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ backup_pvc_namespace }}" + state: absent + force: true diff --git a/roles/backup/tasks/error_handling.yml b/roles/backup/tasks/error_handling.yml new file mode 100644 index 00000000..1d41721d --- /dev/null +++ b/roles/backup/tasks/error_handling.yml @@ -0,0 +1,11 @@ +--- + +- name: Determine the timestamp + set_fact: + now: '{{ lookup("pipe", "date +%FT%TZ") }}' + +- name: Emit ocp event with error + k8s: + kind: Event + namespace: "{{ meta.namespace }}" + template: "event.yml.j2" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml new file mode 100644 index 00000000..42f593d9 --- /dev/null +++ b/roles/backup/tasks/init.yml @@ -0,0 +1,69 @@ +--- + +- name: Delete any existing management pod + k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ backup_pvc_namespace }}" + state: absent + force: true + wait: true + +# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. +- name: Check provided PVC exists + k8s_info: + name: "{{ backup_pvc }}" + kind: PersistentVolumeClaim + namespace: "{{ backup_pvc_namespace }}" + register: provided_pvc + when: + - backup_pvc != '' + +- name: Surface error to user + block: + - name: Set error message + set_fact: + error_msg: "{{ backup_pvc }} does not exist, please create this pvc first." + + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ backup_pvc }} does not exist, please create this pvc first." + when: + - backup_pvc != '' + - provided_pvc.resources | length == 0 + +# If backup_pvc is defined, use in management-pod.yml.j2 +- name: Set default pvc name + set_fact: + _default_backup_pvc: "{{ deployment_name }}-backup-claim" + +# by default, it will re-use the old pvc if already created (unless a pvc is provided) +- name: Set PVC to use for backup + set_fact: + backup_claim: "{{ backup_pvc | default(_default_backup_pvc, true) }}" + +- name: Create PVC for backup + k8s: + kind: PersistentVolumeClaim + template: "backup_pvc.yml.j2" + when: + - backup_pvc == '' or backup_pvc is not defined + +- name: Create management pod from templated deployment config + k8s: + name: "{{ meta.name }}-db-management" + kind: Deployment + state: present + template: "management-pod.yml.j2" + wait: true + +- name: Look up details for this deployment + k8s_info: + api_version: "{{ api_version }}" + kind: "AWX" + name: "{{ deployment_name }}" + namespace: "{{ meta.namespace }}" + register: this_awx diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml new file mode 100644 index 00000000..0ea4ce38 --- /dev/null +++ b/roles/backup/tasks/main.yml @@ -0,0 +1,32 @@ +--- + +- name: Look up details for this backup object + k8s_info: + api_version: "{{ api_version }}" + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + register: this_backup + +- block: + - include_tasks: init.yml + + - include_tasks: postgres.yml + + - include_tasks: secrets.yml + + - include_tasks: awx-cro.yml + + - name: Set flag signifying this backup was successful + set_fact: + backup_complete: true + + - include_tasks: cleanup.yml + + when: + - this_backup['resources'][0]['status']['backupDirectory'] is not defined + +- name: Update status variables + include_tasks: update_status.yml + +# TODO: backup tower settings or make sure that users only specify settings/config changes via AWX object. See ticket diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml new file mode 100644 index 00000000..96e8c588 --- /dev/null +++ b/roles/backup/tasks/postgres.yml @@ -0,0 +1,95 @@ +--- + +- name: Get PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: "{{ this_awx['resources'][0]['status']['towerPostgresConfigurationSecret'] }}" + register: pg_config + +- name: Fail if postgres configuration secret status does not exist + fail: + msg: "The towerPostgresConfigurationSecret status is not set on the AWX object yet or the secret has been deleted." + when: not pg_config | default([]) | length + +- name: Store Database Configuration + set_fact: + awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + awx_postgres_type: "{{ pg_config['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" + +- name: Default label selector to custom resource generated postgres + set_fact: + postgres_label_selector: "app.kubernetes.io/name={{ deployment_name }}-postgres" + when: postgres_label_selector is not defined + +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "{{ postgres_label_selector }}" + register: postgres_pod + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 + +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" + +- name: Determine the timestamp for the backup once for all nodes + set_fact: + now: '{{ lookup("pipe", "date +%F-%T") }}' + +- name: Set backup directory name + set_fact: + backup_dir: "/backups/tower-openshift-backup-{{ now }}" + +- name: Create directory for backup + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + mkdir -p {{ backup_dir }} + +- name: Precreate file for database dump + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + touch {{ backup_dir }}/tower.db + +- name: Set permissions on file for database dump + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "chmod 0600 {{ backup_dir }}/tower.db && chown postgres:root {{ backup_dir }}/tower.db" + +- name: Set full resolvable host name for postgres pod + set_fact: + resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" + when: awx_postgres_type == 'managed' + +- name: Set pg_dump command + set_fact: + pgdump: >- + pg_dump --clean --create + -h {{ resolvable_db_host }} + -U {{ awx_postgres_user }} + -d {{ awx_postgres_database }} + -p {{ awx_postgres_port }} + +- name: Write pg_dump to backup on PVC + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "PGPASSWORD={{ awx_postgres_pass }} {{ pgdump }} > {{ backup_dir }}/tower.db" + register: data_migration diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml new file mode 100644 index 00000000..a93483e4 --- /dev/null +++ b/roles/backup/tasks/secrets.yml @@ -0,0 +1,61 @@ +--- + +- name: Get secret_key + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: "{{ this_awx['resources'][0]['status']['towerSecretKeySecret'] }}" + register: _secret_key + +- name: Set secret key + set_fact: + secret_key: "{{ _secret_key['resources'][0]['data']['secret_key'] | b64decode }}" + +- name: Get admin_password + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: "{{ this_awx['resources'][0]['status']['towerAdminPasswordSecret'] }}" + register: _admin_password + +- name: Set admin_password + set_fact: + admin_password: "{{ _admin_password['resources'][0]['data']['password'] | b64decode }}" + +- name: Get broadcast_websocket + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: "{{ this_awx['resources'][0]['status']['towerBroadcastWebsocketSecret'] }}" + register: _broadcast_websocket + +- name: Set broadcast_websocket key + set_fact: + broadcast_websocket: "{{ _broadcast_websocket['resources'][0]['data']['secret'] | b64decode }}" + +- name: Get postgres configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: "{{ this_awx['resources'][0]['status']['towerPostgresConfigurationSecret'] }}" + register: _postgres_configuration + +- name: Set postgres configuration + set_fact: + database_password: "{{ _postgres_configuration['resources'][0]['data']['password'] | b64decode }}" + database_username: "{{ _postgres_configuration['resources'][0]['data']['username'] | b64decode }}" + database_name: "{{ _postgres_configuration['resources'][0]['data']['database'] | b64decode }}" + database_port: "{{ _postgres_configuration['resources'][0]['data']['port'] | b64decode }}" + database_host: "{{ _postgres_configuration['resources'][0]['data']['host'] | b64decode }}" + database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" + +- name: Template secrets into yaml + set_fact: + secrets_file: "{{ lookup('template', 'secrets.yml.j2') }}" + +- name: Write postgres configuration to pvc + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ secrets_file }}' > {{ backup_dir }}/secrets.yml" diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml new file mode 100644 index 00000000..a497e86a --- /dev/null +++ b/roles/backup/tasks/update_status.yml @@ -0,0 +1,13 @@ +--- + +# The backup directory in this status can be referenced when restoring +- name: Update Tower Backup status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + backupDirectory: "{{ backup_dir }}" + backupClaim: "{{ backup_claim }}" + when: backup_complete diff --git a/roles/backup/templates/backup_pvc.yml.j2 b/roles/backup/templates/backup_pvc.yml.j2 new file mode 100644 index 00000000..d024a565 --- /dev/null +++ b/roles/backup/templates/backup_pvc.yml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ deployment_name }}-backup-claim + namespace: {{ backup_pvc_namespace }} +spec: + accessModes: + - ReadWriteOnce +{% if backup_storage_class != '' %} + storageClassName: {{ backup_storage_class }} +{% endif %} + resources: + requests: + storage: {{ backup_storage_requirements | default('5Gi', true) }} diff --git a/roles/backup/templates/event.yml.j2 b/roles/backup/templates/event.yml.j2 new file mode 100644 index 00000000..ead6aea4 --- /dev/null +++ b/roles/backup/templates/event.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Event +metadata: + name: backup-error.{{ now }} + namespace: {{ meta.namespace }} +involvedObject: + apiVersion: awx.ansible.com/v1beta1 + kind: {{ kind }} + name: {{ meta.name }} + namespace: {{ meta.namespace }} +message: {{ error_msg }} +reason: BackupFailed +type: Warning +firstTimestamp: {{ now }} +lastTimestamp: {{ now }} +count: 1 diff --git a/roles/backup/templates/management-pod.yml.j2 b/roles/backup/templates/management-pod.yml.j2 new file mode 100644 index 00000000..9aa99a1c --- /dev/null +++ b/roles/backup/templates/management-pod.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ meta.name }}-db-management + namespace: {{ backup_pvc_namespace }} +spec: + containers: + - name: {{ meta.name }}-db-management + image: "{{ postgres_image }}" + imagePullPolicy: Always + command: ["sleep", "infinity"] + volumeMounts: + - name: {{ meta.name }}-backup + mountPath: /backups + readOnly: false + volumes: + - name: {{ meta.name }}-backup + persistentVolumeClaim: + claimName: {{ backup_claim }} + readOnly: false + restartPolicy: Never diff --git a/roles/backup/templates/secrets.yml.j2 b/roles/backup/templates/secrets.yml.j2 new file mode 100644 index 00000000..94bacea2 --- /dev/null +++ b/roles/backup/templates/secrets.yml.j2 @@ -0,0 +1,10 @@ +--- +secret_key: {{ secret_key }} +admin_password: {{ admin_password }} +broadcast_websocket: {{ broadcast_websocket }} +database_password: {{ database_password }} +database_username: {{ database_username }} +database_name: {{ database_name }} +database_port: {{ database_port }} +database_host: {{ database_host }} +database_type: {{ database_type }} diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml new file mode 100644 index 00000000..acf8ceef --- /dev/null +++ b/roles/backup/vars/main.yml @@ -0,0 +1,4 @@ +--- +deployment_type: "awx" +postgres_image: postgres:12 +backup_complete: false diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index ec476f7c..acf55554 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -64,6 +64,10 @@ set_fact: pg_config: '{{ _generated_pg_config_resources["resources"] | default([]) | length | ternary(_generated_pg_config_resources, _pg_config) }}' +- name: Set actual postgres configuration secret used + set_fact: + postgres_configuration_secret: "{{ pg_config['resources'][0]['metadata']['name'] }}" + - block: - name: Create Database if no database is specified k8s: @@ -100,7 +104,6 @@ definition: "{{ lookup('template', 'tower_postgres.yaml.j2') }}" when: pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' - - name: Store Database Configuration set_fact: awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" @@ -112,8 +115,8 @@ - name: Look up details for this deployment k8s_info: - api_version: 'v1beta1' # TODO: How to parameterize this? - kind: "AWX" # TODO: How to parameterize this? + api_version: "{{ api_version }}" + kind: "{{ kind }}" name: "{{ meta.name }}" namespace: "{{ meta.namespace }}" register: this_awx diff --git a/roles/installer/tasks/initialize_django.yml b/roles/installer/tasks/initialize_django.yml index 62849d7e..3e977e06 100644 --- a/roles/installer/tasks/initialize_django.yml +++ b/roles/installer/tasks/initialize_django.yml @@ -1,20 +1,31 @@ --- - name: Check if there are any super users defined. - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" command: >- bash -c "echo 'from django.contrib.auth.models import User; - nsu = User.objects.filter(is_superuser=True).count(); + nsu = User.objects.filter(is_superuser=True, username='{{ tower_admin_user }}').count(); exit(0 if nsu > 0 else 1)' | awx-manage shell" ignore_errors: true register: users_result changed_when: users_result.return_code > 0 +- name: Update super user password via Django if it does exist (same password is a noop) + k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ tower_pod_name }}" + container: "{{ meta.name }}-task" + command: >- + bash -c "awx-manage update_password --username '{{ tower_admin_user }}' --password '{{ tower_admin_password }}'" + register: update_pw_result + changed_when: users_result.stdout == 'Password not updated' + when: users_result.return_code == 0 + - name: Create super user via Django if it doesn't exist. - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" @@ -25,7 +36,7 @@ when: users_result.return_code > 0 - name: Create preload data if necessary. # noqa 305 - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" diff --git a/roles/installer/tasks/load_ldap_cacert_secret.yml b/roles/installer/tasks/load_ldap_cacert_secret.yml index 41667a1b..ebf5fcc2 100644 --- a/roles/installer/tasks/load_ldap_cacert_secret.yml +++ b/roles/installer/tasks/load_ldap_cacert_secret.yml @@ -1,6 +1,6 @@ --- - name: Retrieve LDAP CA Certificate Secret - community.kubernetes.k8s_info: + k8s_info: kind: Secret namespace: '{{ meta.namespace }}' name: '{{ ldap_cacert_secret }}' diff --git a/roles/installer/tasks/load_route_tls_secret.yml b/roles/installer/tasks/load_route_tls_secret.yml index 03b50226..529e5851 100644 --- a/roles/installer/tasks/load_route_tls_secret.yml +++ b/roles/installer/tasks/load_route_tls_secret.yml @@ -1,6 +1,6 @@ --- - name: Retrieve Route TLS Secret - community.kubernetes.k8s_info: + k8s_info: kind: Secret namespace: '{{ meta.namespace }}' name: '{{ tower_route_tls_secret }}' diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index aeea94ec..1f83d8f3 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -1,12 +1,21 @@ --- +- name: Set actual old postgres configuration secret name + set_fact: + old_postgres_configuration_name: "{{ old_pg_config['resources'][0]['metadata']['name'] }}" + - name: Store Database Configuration set_fact: - tower_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}" - tower_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}" - tower_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}" - tower_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}" - tower_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}" + awx_old_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_old_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_old_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_old_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_old_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}" + +- name: Default label selector to custom resource generated postgres + set_fact: + postgres_label_selector: "app.kubernetes.io/name={{ meta.name }}-postgres" + when: postgres_label_selector is not defined - name: Get the postgres pod information k8s_info: @@ -16,7 +25,9 @@ field_selectors: - status.phase=Running register: postgres_pod - until: postgres_pod['resources'] | length + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" delay: 5 retries: 60 @@ -31,10 +42,10 @@ set_fact: pgdump: >- pg_dump --clean --create - -h {{ tower_old_postgres_host }} - -U {{ tower_old_postgres_user }} - -d {{ tower_old_postgres_database }} - -p {{ tower_old_postgres_port }} + -h {{ awx_old_postgres_host }} + -U {{ awx_old_postgres_user }} + -d {{ awx_old_postgres_database }} + -p {{ awx_old_postgres_port }} - name: Set pg_restore command set_fact: @@ -44,13 +55,13 @@ -p {{ awx_postgres_port }} - name: Stream backup from pg_dump to the new postgresql container - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ postgres_pod_name }}" command: | bash -c """ set -e -o pipefail - PGPASSWORD={{ tower_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} + PGPASSWORD={{ awx_old_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} echo 'Successful' """ register: data_migration @@ -58,4 +69,4 @@ - name: Set flag signifying that this instance has been migrated set_fact: - tower_migrated_from_secret: "{{ tower_old_postgres_configuration_secret }}" + tower_migrated_from_secret: "{{ old_postgres_configuration_name }}" diff --git a/roles/installer/tasks/update_status.yml b/roles/installer/tasks/update_status.yml index ec4b3d54..c3194992 100644 --- a/roles/installer/tasks/update_status.yml +++ b/roles/installer/tasks/update_status.yml @@ -17,8 +17,35 @@ status: towerAdminUser: "{{ tower_admin_user }}" +- name: Update postgres configuration status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerPostgresConfigurationSecret: "{{ pg_config['resources'][0]['metadata']['name'] }}" + +- name: Update broadcast websocket status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerBroadcastWebsocketSecret: "{{ broadcast_websocket_secret['resources'][0]['metadata']['name'] }}" + +- name: Update secret key status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerSecretKeySecret: "{{ secret_key_secret_name }}" + - name: Retrieve instance version - community.kubernetes.k8s_exec: + k8s_exec: namespace: "{{ meta.namespace }}" pod: "{{ tower_pod_name }}" container: "{{ meta.name }}-task" @@ -47,7 +74,7 @@ - block: - name: Retrieve route URL - community.kubernetes.k8s_info: + k8s_info: kind: Route namespace: '{{ meta.namespace }}' name: '{{ meta.name }}' diff --git a/roles/installer/templates/tower_postgres.yaml.j2 b/roles/installer/templates/tower_postgres.yaml.j2 index a98086a8..694a6940 100644 --- a/roles/installer/templates/tower_postgres.yaml.j2 +++ b/roles/installer/templates/tower_postgres.yaml.j2 @@ -53,17 +53,17 @@ spec: - name: POSTGRES_DB valueFrom: secretKeyRef: - name: '{{ meta.name }}-postgres-configuration' + name: '{{ postgres_configuration_secret }}' key: database - name: POSTGRES_USER valueFrom: secretKeyRef: - name: '{{ meta.name }}-postgres-configuration' + name: '{{ postgres_configuration_secret }}' key: username - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: '{{ meta.name }}-postgres-configuration' + name: '{{ postgres_configuration_secret }}' key: password - name: PGDATA value: '{{ tower_postgres_data_path }}' @@ -72,7 +72,7 @@ spec: - name: POSTGRES_HOST_AUTH_METHOD value: '{{ postgres_host_auth_method }}' ports: - - containerPort: 5432 + - containerPort: {{ awx_postgres_port | default('5432')}} name: postgres volumeMounts: - name: postgres diff --git a/roles/restore/.secrets.yml.swp b/roles/restore/.secrets.yml.swp new file mode 100644 index 00000000..32096989 Binary files /dev/null and b/roles/restore/.secrets.yml.swp differ diff --git a/roles/restore/README.md b/roles/restore/README.md new file mode 100644 index 00000000..0ba8580b --- /dev/null +++ b/roles/restore/README.md @@ -0,0 +1,121 @@ +Restore Role +========= + +The purpose of this role is to restore your AWX deployment from an existing PVC backup. The backup includes: + - custom deployment specific values in the spec section of the AWX custom resource object + - backup of the postgresql database + - secret_key, admin_password, and broadcast_websocket secrets + - database configuration + + + +Requirements +------------ + +This role assumes you are authenticated with an Openshift or Kubernetes cluster: + - The awx-operator has been deployed to the cluster + - AWX is deployed to via the operator + - An AWX backup is available on a PVC in your cluster (see the backup [README.md](../backup/README.md)) + + +Usage +---------------- + +Then create a file named `restore-awx.yml` with the following contents: + +```yaml +--- +apiVersion: awx.ansible.com/v1beta1 +kind: AWXRestore +metadata: + name: restore1 + namespace: my-namespace +spec: + deployment_name: mytower + backup: awxbackup-2021-04-22 + backup_pvc_namespace: 'old-awx-namespace' +``` + +Note that the `deployment_name` above is the name of the AWX deployment you intend to create and restore to. + +The namespace specified is the namespace the resulting AWX deployment will be in. The namespace you specified must be pre-created. + +``` +kubectl create ns my-namespace +``` + +Finally, use `kubectl` to create the restore object in your cluster: + +```bash +$ kubectl apply -f restore-awx.yml +``` + +This will create a new deployment and restore your backup to it. + +> :warning: tower_admin_password_secret value will replace the password for the `tower_admin_user` user (by default, this is the `admin` user). + + +Role Variables +-------------- + +The name of the backup directory can be found as a status on your AWXBackup object. This can be found in your cluster's console, or with the client as shown below. + +```bash +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.backupDirectory}" +/backups/tower-openshift-backup-2021-04-02-03:25:08 +``` + +``` +backup_dir: '/backups/tower-openshift-backup-2021-04-02-03:25:08' +``` + + +The name of the PVC can also be found by looking at the backup object. + +```bash +$ kubectl get awxbackup awxbackup1 -o jsonpath="{.items[0].status.backupClaim}" +awx-backup-volume-claim +``` + +``` +backup_pvc: 'awx-backup-volume-claim' +``` + +By default, the backup pvc will be created in the same namespace the awxbackup object is created in. This namespace must be specified using the `backup_pvc_namespace` variable. + +``` +backup_pvc_namespace: 'custom-namespace' +``` + +If a custom postgres configuration secret was used when deploying AWX, it must be set: + +``` +tower_postgres_configuration_secret: 'awx-postgres-configuration' +``` + +If the awxbackup object no longer exists, it is still possible to restore from the backup it created by specifying the pvc name and the back directory. + +``` +backup_pvc: myoldtower-backup-claim +backup_dir: /backups/tower-openshift-backup-2021-04-02-03:25:08 +``` + + +Testing +---------------- + +You can test this role directly by creating and running the following playbook with the appropriate variables: + +``` +--- +- name: Restore AWX + hosts: localhost + gather_facts: false + roles: + - restore +``` + +License +------- + +MIT diff --git a/roles/restore/defaults/main.yml b/roles/restore/defaults/main.yml new file mode 100644 index 00000000..0a9422a9 --- /dev/null +++ b/roles/restore/defaults/main.yml @@ -0,0 +1,14 @@ +--- +# Required: specify name of tower deployment to restore to +deployment_name: '' +kind: 'AWXRestore' +api_version: '{{ deployment_type }}.ansible.com/v1beta1' + +# Required: specify a pre-created PVC (name) to restore from +backup_pvc: '' +backup_pvc_namespace: '' + +# Required: backup name, found on the awxbackup object +backup_dir: '' + +tower_postgres_configuration_secret: "{{ deployment_name }}-postgres-configuration" diff --git a/roles/restore/meta/main.yml b/roles/restore/meta/main.yml new file mode 100644 index 00000000..720b9bcb --- /dev/null +++ b/roles/restore/meta/main.yml @@ -0,0 +1,31 @@ +--- +galaxy_info: + author: Ansible + description: AWX role for AWX Operator for Kubernetes. + company: Red Hat, Inc. + + license: MIT + + min_ansible_version: 2.8 + + platforms: + - name: EL + versions: + - all + - name: Debian + versions: + - all + + galaxy_tags: + - tower + - controller + - awx + - ansible + - restore + - automation + +dependencies: [] + +collections: + - community.kubernetes + - operator_sdk.util diff --git a/roles/restore/tasks/cleanup.yml b/roles/restore/tasks/cleanup.yml new file mode 100644 index 00000000..94a545be --- /dev/null +++ b/roles/restore/tasks/cleanup.yml @@ -0,0 +1,24 @@ +--- + +- name: Delete any existing management pod + k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ backup_pvc_namespace }}" + state: absent + force: true + +- name: Remove ownerReferences from secrets to avoid garbage collection + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: '{{ item }}' + namespace: '{{ meta.namespace }}' + ownerReferences: null + loop: + - '{{ deployment_name }}-admin-password' + - '{{ deployment_name }}-secret-key' + - '{{ deployment_name }}-postgres-configuration' + - '{{ deployment_name }}-broadcast-websocket' diff --git a/roles/restore/tasks/deploy_awx.yml b/roles/restore/tasks/deploy_awx.yml new file mode 100644 index 00000000..5fc1b7fc --- /dev/null +++ b/roles/restore/tasks/deploy_awx.yml @@ -0,0 +1,38 @@ +--- + +- name: Get AWX object definition from pvc + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "cat '{{ backup_dir }}/awx_object'" + register: awx_object + +- name: Set AWX spec variable from backup + set_fact: + awx_spec: "{{ awx_object.stdout }}" + +- name: Deploy AWX + k8s: + state: "{{ state | default('present') }}" + namespace: "{{ meta.namespace }}" + apply: yes + template: awx_object.yml.j2 + wait: true + wait_condition: + type: "Running" + status: "True" + +# TODO: Add logic to allow users to provide override values here, +# or to specify spec values that were not in the backed up AWX object. +# This may involve changing how we back up the spec section of the AWX object + +- name: Remove ownerReferences to prevent garbage collection of new AWX CRO + k8s: + definition: + apiVersion: '{{ api_version }}' + kind: AWX + metadata: + name: '{{ deployment_name }}' + namespace: '{{ meta.namespace }}' + ownerReferences: null diff --git a/roles/restore/tasks/error_handling.yml b/roles/restore/tasks/error_handling.yml new file mode 100644 index 00000000..1d41721d --- /dev/null +++ b/roles/restore/tasks/error_handling.yml @@ -0,0 +1,11 @@ +--- + +- name: Determine the timestamp + set_fact: + now: '{{ lookup("pipe", "date +%FT%TZ") }}' + +- name: Emit ocp event with error + k8s: + kind: Event + namespace: "{{ meta.namespace }}" + template: "event.yml.j2" diff --git a/roles/restore/tasks/init.yml b/roles/restore/tasks/init.yml new file mode 100644 index 00000000..9b16e9cd --- /dev/null +++ b/roles/restore/tasks/init.yml @@ -0,0 +1,88 @@ +--- + +- name: Set variables from awxbackup object statuses if provided + block: + - name: Look up details for the backup object + k8s_info: + api_version: "{{ api_version }}" + kind: "AWXBackup" + name: "{{ backup }}" + namespace: "{{ backup_pvc_namespace }}" + register: this_backup + + - name: Set backup pvc name from status + set_fact: + backup_pvc: "{{ this_backup['resources'][0]['status']['backupClaim'] }}" + + - name: Set tmp backup directory from status + set_fact: + backup_dir: "{{ this_backup['resources'][0]['status']['backupDirectory'] }}" + when: + - backup != '' or backup is defined + +# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. +- name: Check provided PVC exists + k8s_info: + name: "{{ backup_pvc }}" + kind: PersistentVolumeClaim + namespace: "{{ backup_pvc_namespace }}" + register: provided_pvc + when: + - backup_pvc != '' + +- name: Surface error to user + block: + - name: Set error message + set_fact: + error_msg: "{{ backup_pvc }} does not exist, please create this pvc first." + + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ error_msg }}" + when: + - backup_pvc != '' + - provided_pvc.resources | length == 0 + +- name: Delete any existing management pod + k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ backup_pvc_namespace }}" + state: absent + force: true + wait: true + +- name: Create management pod from templated deployment config + k8s: + name: "{{ meta.name }}-db-management" + kind: Deployment + state: present + template: "management-pod.yml.j2" + wait: true + +- name: Check to make sure backup directory exists on PVC + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "stat {{ backup_dir }}" + register: stat_backup_dir + +- name: Error if backup dir is missing + block: + - name: Set error message + set_fact: + error_msg: "{{ backup_dir }} does not exist, see the backupDirectory status on your AWXBackup for the correct backup_dir." + + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if backup dir provided does not exist + fail: + msg: "{{ error_msg }}" + when: + - backup_dir != '' + - stat_backup_dir.return_code != 0 diff --git a/roles/restore/tasks/main.yml b/roles/restore/tasks/main.yml new file mode 100644 index 00000000..2710b2aa --- /dev/null +++ b/roles/restore/tasks/main.yml @@ -0,0 +1,32 @@ +--- + +- name: Look up details for this restore object + k8s_info: + api_version: "{{ api_version }}" + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + register: this_restore + +- block: + - include_tasks: init.yml + + - include_tasks: secrets.yml + + - include_tasks: deploy_awx.yml + + - include_tasks: postgres.yml + + - name: Set flag signifying this restore was successful + set_fact: + tower_restore_complete: True + + - include_tasks: cleanup.yml + + when: + - this_restore['resources'][0]['status']['towerRestoreComplete'] is not defined + +- name: Update status variables + include_tasks: update_status.yml + +# TODO: backup tower settings or make sure that users only specify settings/config changes via AWX object. See ticket diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml new file mode 100644 index 00000000..967cd77c --- /dev/null +++ b/roles/restore/tasks/postgres.yml @@ -0,0 +1,95 @@ +--- + +- name: Check for specified PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ tower_postgres_configuration_secret }}' + register: _custom_pg_config_resources + when: tower_postgres_configuration_secret | length + +- name: Check for default PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ deployment_name }}-postgres-configuration' + register: _default_pg_config_resources + +- name: Set PostgreSQL configuration + set_fact: + pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' + +- name: Store Database Configuration + set_fact: + awx_postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + awx_postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + awx_postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + awx_postgres_type: "{{ pg_config['resources'][0]['data']['type'] | b64decode | default('unmanaged') }}" + +- name: Default label selector to custom resource generated postgres + set_fact: + postgres_label_selector: "app.kubernetes.io/name={{ deployment_name }}-postgres" + when: postgres_label_selector is not defined + +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "{{ postgres_label_selector }}" + register: postgres_pod + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 + +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" + +- name: Check for presence of AWX Deployment + k8s_info: + api_version: v1 + kind: Deployment + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + register: this_deployment + +- name: Scale down Deployment for migration + k8s_scale: + api_version: v1 + kind: Deployment + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + replicas: 0 + when: this_deployment['resources'] | length + +- name: Set full resolvable host name for postgres pod + set_fact: + resolvable_db_host: "{{ awx_postgres_host }}.{{ meta.namespace }}.svc.cluster.local" + when: awx_postgres_type == 'managed' + +- name: Set pg_restore command + set_fact: + psql_restore: >- + psql -U {{ awx_postgres_user }} + -h {{ resolvable_db_host }} + -U {{ awx_postgres_user }} + -d {{ awx_postgres_database }} + -p {{ awx_postgres_port }} + +- name: Restore database dump to the new postgresql container + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: | + bash -c """ + set -e -o pipefail + cat {{ backup_dir }}/tower.db | PGPASSWORD={{ awx_postgres_pass }} {{ psql_restore }} + echo 'Successful' + """ + register: data_migration + failed_when: "'Successful' not in data_migration.stdout" diff --git a/roles/restore/tasks/secrets.yml b/roles/restore/tasks/secrets.yml new file mode 100644 index 00000000..b543769a --- /dev/null +++ b/roles/restore/tasks/secrets.yml @@ -0,0 +1,37 @@ +--- + +- name: Get secret definition from pvc + k8s_exec: + namespace: "{{ backup_pvc_namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "cat '{{ backup_dir }}/secrets.yml'" + register: secrets + +- name: Create temp vars file + tempfile: + prefix: secret_vars- + register: secret_vars + +- name: Write vars to file locally + copy: + dest: "{{ secret_vars.path }}" + content: "{{ secrets.stdout }}" + mode: 0640 + +- name: Include secret vars from backup + include_vars: "{{ secret_vars.path }}" + +- name: Set new database host based on supplied deployment_name + set_fact: + database_host: "{{ deployment_name }}-postgres" + when: + - database_type == 'managed' + +- name: Apply secret + k8s: + state: present + namespace: "{{ meta.namespace }}" + apply: yes + wait: yes + template: "secrets.yml.j2" diff --git a/roles/restore/tasks/update_status.yml b/roles/restore/tasks/update_status.yml new file mode 100644 index 00000000..2b63a884 --- /dev/null +++ b/roles/restore/tasks/update_status.yml @@ -0,0 +1,11 @@ +--- + +- name: Update Tower Restore status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + towerRestoreComplete: true + when: tower_restore_complete is defined diff --git a/roles/restore/templates/awx_object.yml.j2 b/roles/restore/templates/awx_object.yml.j2 new file mode 100644 index 00000000..53f01326 --- /dev/null +++ b/roles/restore/templates/awx_object.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: '{{ api_version }}' +kind: AWX +metadata: + name: '{{ deployment_name }}' + namespace: '{{ meta.namespace }}' +spec: {{ awx_spec }} diff --git a/roles/restore/templates/event.yml.j2 b/roles/restore/templates/event.yml.j2 new file mode 100644 index 00000000..3670cba3 --- /dev/null +++ b/roles/restore/templates/event.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Event +metadata: + name: restore-error.{{ now }} + namespace: {{ meta.namespace }} +involvedObject: + apiVersion: awx.ansible.com/v1beta1 + kind: {{ kind }} + name: {{ meta.name }} + namespace: {{ meta.namespace }} +message: {{ error_msg }} +reason: RestoreFailed +type: Warning +firstTimestamp: {{ now }} +lastTimestamp: {{ now }} +count: 1 diff --git a/roles/restore/templates/management-pod.yml.j2 b/roles/restore/templates/management-pod.yml.j2 new file mode 100644 index 00000000..301bbfbb --- /dev/null +++ b/roles/restore/templates/management-pod.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ meta.name }}-db-management + namespace: {{ backup_pvc_namespace }} +spec: + containers: + - name: {{ meta.name }}-db-management + image: "{{ postgres_image }}" + imagePullPolicy: Always + command: ["sleep", "infinity"] + volumeMounts: + - name: {{ meta.name }}-backup + mountPath: /backups + readOnly: false + volumes: + - name: {{ meta.name }}-backup + persistentVolumeClaim: + claimName: {{ backup_pvc }} + readOnly: false + restartPolicy: Never diff --git a/roles/restore/templates/secrets.yml.j2 b/roles/restore/templates/secrets.yml.j2 new file mode 100644 index 00000000..4d718421 --- /dev/null +++ b/roles/restore/templates/secrets.yml.j2 @@ -0,0 +1,44 @@ +# Postgres Secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: '{{ deployment_name }}-postgres-configuration' + namespace: '{{ meta.namespace }}' +stringData: + password: '{{ database_password }}' + username: '{{ database_username }}' + database: '{{ database_name }}' + port: '{{ database_port }}' + host: '{{ database_host }}' + type: '{{ database_type }}' + +# Secret Key Secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: '{{ deployment_name }}-secret-key' + namespace: '{{ meta.namespace }}' +stringData: + secret_key: '{{ secret_key }}' + +# Admin Password Secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: '{{ deployment_name }}-admin-password' + namespace: '{{ meta.namespace }}' +stringData: + password: '{{ admin_password }}' + +# Broadcast Websocket Secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: '{{ deployment_name }}-broadcast-websocket' + namespace: '{{ meta.namespace }}' +stringData: + secret: '{{ broadcast_websocket }}' diff --git a/roles/restore/vars/main.yml b/roles/restore/vars/main.yml new file mode 100644 index 00000000..c2a30003 --- /dev/null +++ b/roles/restore/vars/main.yml @@ -0,0 +1,4 @@ +--- + +deployment_type: "awx" +postgres_image: postgres:12 diff --git a/watches.yaml b/watches.yaml index 02e2eb99..0a5f795b 100644 --- a/watches.yaml +++ b/watches.yaml @@ -6,3 +6,13 @@ finalizer: name: finalizer.awx.ansible.com role: finalizer + +- version: v1beta1 + group: awx.ansible.com + kind: AWXBackup + role: backup + +- version: v1beta1 + group: awx.ansible.com + kind: AWXRestore + role: restore