From 39ef816f5e8659194a6922e18acdb7723a7e5f21 Mon Sep 17 00:00:00 2001 From: Marcelo Moreira de Mello Date: Thu, 15 Apr 2021 23:49:31 -0400 Subject: [PATCH] Updated logic to avoid crash during upgrades --- .../tasks/database_configuration.yml | 24 ++++ .../{initialize.yml => initialize_django.yml} | 0 roles/installer/tasks/main.yml | 46 +------- roles/installer/tasks/migrate_data.yml | 7 +- .../tasks/resources_configuration.yml | 107 ++++++++++++++++++ .../templates/tower_postgres.yaml.j2 | 2 +- roles/installer/vars/main.yml | 1 + 7 files changed, 141 insertions(+), 46 deletions(-) rename roles/installer/tasks/{initialize.yml => initialize_django.yml} (100%) create mode 100644 roles/installer/tasks/resources_configuration.yml diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index a79f55c4..af40c024 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -64,12 +64,36 @@ set_fact: pg_config: '{{ _generated_pg_config_resources["resources"] | default([]) | length | ternary(_generated_pg_config_resources, _pg_config) }}' +- name: Check if PostgreSQL statefuset is present + k8s_info: + api_version: v1 + kind: StatefulSet + namespace: '{{ meta.namespace }}' + name: '{{ meta.name }}-postgres' + register: _postgres_statefulset_result + +- name: Set manual_statefulset_upgrade to false since PostgreSQL statefulset does not exist yet + set_fact: + manual_statefulset_upgrade: False + when: not _postgres_statefulset_result['resources'] | length + +- name: Determine if PostgreSQL statefulset requires manual upgrade + set_fact: + manual_statefulset_upgrade: '{{ _postgres_statefulset_result["resources"][0]["metadata"]["labels"]["app.kubernetes.io/managed-by"] | default("") | ternary("False", "True") }}' + when: manual_statefulset_upgrade is not defined + +- name: Show help message since PostgreSQL statefulset was created by a legacy version + debug: + var: manual_upgrade_msg + when: manual_statefulset_upgrade|bool + - name: Create Database if no database is specified k8s: apply: true definition: "{{ lookup('template', 'tower_postgres.yaml.j2') }}" when: - pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' + - not manual_statefulset_upgrade|bool - name: Store Database Configuration set_fact: diff --git a/roles/installer/tasks/initialize.yml b/roles/installer/tasks/initialize_django.yml similarity index 100% rename from roles/installer/tasks/initialize.yml rename to roles/installer/tasks/initialize_django.yml diff --git a/roles/installer/tasks/main.yml b/roles/installer/tasks/main.yml index 1d2bc7c5..9fad33b9 100644 --- a/roles/installer/tasks/main.yml +++ b/roles/installer/tasks/main.yml @@ -43,47 +43,8 @@ - tower_ingress_type | lower == 'route' - tower_route_tls_secret != '' -- name: Ensure configured instance resources exist in the cluster. - k8s: - apply: yes - definition: "{{ lookup('template', item) | from_yaml_all | list }}" - with_items: - - tower_config.yaml.j2 - -- name: Apply Resources - k8s: - apply: yes - definition: "{{ lookup('template', item + '.yaml.j2') }}" - register: tower_deployment_result - loop: - - 'tower_app_credentials' - - 'tower_service_account' - - 'tower_persistent' - - 'tower_deployment' - - 'tower_service' - - 'tower_ingress' - -- name: Get the resource pod information. - k8s_info: - kind: Pod - namespace: '{{ meta.namespace }}' - label_selectors: - - "app.kubernetes.io/name={{ meta.name }}" - - "app.kubernetes.io/managed-by=awx-operator" - - "app.kubernetes.io/component=awx" - register: tower_pods - until: "tower_pods['resources'][0]['status']['phase'] == 'Running'" - delay: 5 - retries: 60 - -- name: Set the resource pod name as a variable. - set_fact: - tower_pod_name: "{{ tower_pods['resources'][0]['metadata']['name'] }}" - -- name: Verify the resource pod name is populated. - assert: - that: tower_pod_name != '' - fail_msg: "Could not find the tower pod's name." +- name: Include resources configuration tasks + include_tasks: resources_configuration.yml - name: Check for pending migrations k8s_exec: @@ -107,7 +68,8 @@ - database_check is defined - (database_check.stdout|trim) != '0' -- include_tasks: initialize.yml +- name: Initialize Django + include_tasks: initialize_django.yml - name: Update status variables include_tasks: update_status.yml diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml index cd619af0..da5067c0 100644 --- a/roles/installer/tasks/migrate_data.yml +++ b/roles/installer/tasks/migrate_data.yml @@ -12,10 +12,11 @@ k8s_info: kind: Pod namespace: '{{ meta.namespace }}' - label_selectors: - - "app.kubernetes.io/name={{ meta.name }}-postgres" + name: '{{ meta.name }}-postgres-0' # using name to keep compatibility + field_selectors: + - status.phase=Running register: postgres_pod - until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + until: postgres_pod['resources'] | length delay: 5 retries: 60 diff --git a/roles/installer/tasks/resources_configuration.yml b/roles/installer/tasks/resources_configuration.yml new file mode 100644 index 00000000..90818953 --- /dev/null +++ b/roles/installer/tasks/resources_configuration.yml @@ -0,0 +1,107 @@ +--- + +- name: Check if deployment is present + k8s_info: + api_version: v1 + kind: Deployment + namespace: '{{ meta.namespace }}' + name: '{{ meta.name }}' + register: _deployment_result + +- name: Set manual_deployment_upgrade to false since deployment does not exist yet + set_fact: + manual_deployment_upgrade: False + when: not _deployment_result['resources'] | length + +- name: Determine if deployment requires manual upgrade + set_fact: + manual_deployment_upgrade: '{{ _deployment_result["resources"][0]["metadata"]["labels"]["app.kubernetes.io/managed-by"] | default("") | ternary("False", "True") }}' + when: manual_deployment_upgrade is not defined + +- name: Determine pod label_selector to be used + set_fact: + tower_pod_label_selector: '{{ manual_deployment_upgrade|bool | default("false"|bool)| ternary("app={{ deployment_type }}", "app.kubernetes.io/name={{ meta.name }}") }}' + +- name: Show help message since deployment was created by a legacy version + debug: + var: manual_upgrade_msg + when: manual_deployment_upgrade|bool + +- name: Get the current resource pod information. + k8s_info: + api_version: v1 + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - '{{ tower_pod_label_selector }}' + field_selectors: + - status.phase=Running + register: tower_pods + changed_when: tower_pods["resources"] | length + +- name: Set the resource pod name as a variable. + set_fact: + tower_pod_name: "{{ tower_pods['resources'][0]['metadata']['name'] | default('') }}" + +- name: Apply Resources + k8s: + apply: yes + definition: "{{ lookup('template', item + '.yaml.j2') }}" + wait: yes + register: tower_resources_result + loop: + - 'tower_config' + - 'tower_app_credentials' + - 'tower_service_account' + - 'tower_persistent' + - 'tower_service' + - 'tower_ingress' + when: not manual_deployment_upgrade|bool + +- name: Apply deployment resources + k8s: + apply: yes + definition: "{{ lookup('template', 'tower_deployment.yaml.j2') | from_yaml }}" + wait: yes + register: tower_deployment_result + when: not manual_deployment_upgrade|bool + +- block: + - name: Delete pod to reload configuration + k8s: + api_version: v1 + state: absent + kind: Pod + namespace: '{{ meta.namespace }}' + name: '{{ tower_pod_name }}' + wait: yes + when: + - tower_resources_result.changed + - tower_pod_name | length + + - name: Get the new resource pod information after updating resource. + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - '{{ tower_pod_label_selector }}' + field_selectors: + - status.phase=Running + register: _new_pod + until: + - _new_pod['resources'] | length + - _new_pod['resources'][0]['metadata']['name'] != tower_pod_name + delay: 5 + retries: 60 + when: not manual_deployment_upgrade|bool + + - name: Update new resource pod name as a variable. + set_fact: + tower_pod_name: '{{ _new_pod["resources"][0]["metadata"]["name"] }}' + when: + - tower_resources_result.changed or tower_deployment_result.changed + +- name: Verify the resource pod name is populated. + assert: + that: tower_pod_name != '' + fail_msg: "Could not find the tower pod's name." diff --git a/roles/installer/templates/tower_postgres.yaml.j2 b/roles/installer/templates/tower_postgres.yaml.j2 index 18d2ae96..f60cf29c 100644 --- a/roles/installer/templates/tower_postgres.yaml.j2 +++ b/roles/installer/templates/tower_postgres.yaml.j2 @@ -1,6 +1,6 @@ # Postgres StatefulSet. --- -apiVersion: v1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: '{{ meta.name }}-postgres' diff --git a/roles/installer/vars/main.yml b/roles/installer/vars/main.yml index b454ee74..469912e3 100644 --- a/roles/installer/vars/main.yml +++ b/roles/installer/vars/main.yml @@ -3,3 +3,4 @@ postgres_initdb_args: '--auth-host=scram-sha-256' postgres_host_auth_method: 'scram-sha-256' ldap_cacert_ca_crt: '' tower_projects_existing_claim: '' +manual_upgrade_msg: 'Manual Upgrade is required, please refer to the upgrade docs'