Updated logic to avoid crash during upgrades

roles/installer/tasks/database_configuration.yml

@@ -64,12 +64,36 @@
   set_fact:
     pg_config: '{{ _generated_pg_config_resources["resources"] | default([]) | length | ternary(_generated_pg_config_resources, _pg_config) }}'
 
+- name: Check if PostgreSQL statefuset is present
+  k8s_info:
+    api_version: v1
+    kind: StatefulSet
+    namespace: '{{ meta.namespace }}'
+    name: '{{ meta.name }}-postgres'
+  register: _postgres_statefulset_result
+
+- name: Set manual_statefulset_upgrade to false since PostgreSQL statefulset does not exist yet
+  set_fact:
+    manual_statefulset_upgrade: False
+  when: not _postgres_statefulset_result['resources'] | length
+
+- name: Determine if PostgreSQL statefulset requires manual upgrade
+  set_fact:
+    manual_statefulset_upgrade: '{{ _postgres_statefulset_result["resources"][0]["metadata"]["labels"]["app.kubernetes.io/managed-by"] | default("") | ternary("False", "True") }}'
+  when: manual_statefulset_upgrade is not defined
+
+- name: Show help message since PostgreSQL statefulset was created by a legacy version
+  debug:
+    var: manual_upgrade_msg
+  when: manual_statefulset_upgrade|bool
+
 - name: Create Database if no database is specified
   k8s:
     apply: true
     definition: "{{ lookup('template', 'tower_postgres.yaml.j2') }}"
   when:
     - pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed'
+    - not manual_statefulset_upgrade|bool
 
 - name: Store Database Configuration
   set_fact:

roles/installer/tasks/main.yml

@@ -43,47 +43,8 @@
     - tower_ingress_type | lower == 'route'
     - tower_route_tls_secret != ''
 
-- name: Ensure configured instance resources exist in the cluster.
-  k8s:
-    apply: yes
-    definition: "{{ lookup('template', item) | from_yaml_all | list }}"
-  with_items:
-    - tower_config.yaml.j2
-
-- name: Apply Resources
-  k8s:
-    apply: yes
-    definition: "{{ lookup('template', item + '.yaml.j2') }}"
-  register: tower_deployment_result
-  loop:
-    - 'tower_app_credentials'
-    - 'tower_service_account'
-    - 'tower_persistent'
-    - 'tower_deployment'
-    - 'tower_service'
-    - 'tower_ingress'
-
-- name: Get the resource pod information.
-  k8s_info:
-    kind: Pod
-    namespace: '{{ meta.namespace }}'
-    label_selectors:
-      - "app.kubernetes.io/name={{ meta.name }}"
-      - "app.kubernetes.io/managed-by=awx-operator"
-      - "app.kubernetes.io/component=awx"
-  register: tower_pods
-  until: "tower_pods['resources'][0]['status']['phase'] == 'Running'"
-  delay: 5
-  retries: 60
-
-- name: Set the resource pod name as a variable.
-  set_fact:
-    tower_pod_name: "{{ tower_pods['resources'][0]['metadata']['name'] }}"
-
-- name: Verify the resource pod name is populated.
-  assert:
-    that: tower_pod_name != ''
-    fail_msg: "Could not find the tower pod's name."
+- name: Include resources configuration tasks
+  include_tasks: resources_configuration.yml
 
 - name: Check for pending migrations
   k8s_exec:
@@ -107,7 +68,8 @@
     - database_check is defined
     - (database_check.stdout|trim) != '0'
 
-- include_tasks: initialize.yml
+- name: Initialize Django
+  include_tasks: initialize_django.yml
 
 - name: Update status variables
   include_tasks: update_status.yml

roles/installer/tasks/migrate_data.yml

@@ -12,10 +12,11 @@
   k8s_info:
     kind: Pod
     namespace: '{{ meta.namespace }}'
-    label_selectors:
-      - "app.kubernetes.io/name={{ meta.name }}-postgres"
+    name: '{{ meta.name }}-postgres-0'  # using name to keep compatibility
+    field_selectors:
+      - status.phase=Running
   register: postgres_pod
-  until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  until: postgres_pod['resources'] | length
   delay: 5
   retries: 60
 

roles/installer/tasks/resources_configuration.yml

@@ -0,0 +1,107 @@
+---
+
+- name: Check if deployment is present
+  k8s_info:
+    api_version: v1
+    kind: Deployment
+    namespace: '{{ meta.namespace }}'
+    name: '{{ meta.name }}'
+  register: _deployment_result
+
+- name: Set manual_deployment_upgrade to false since deployment does not exist yet
+  set_fact:
+    manual_deployment_upgrade: False
+  when: not _deployment_result['resources'] | length
+
+- name: Determine if deployment requires manual upgrade
+  set_fact:
+    manual_deployment_upgrade: '{{ _deployment_result["resources"][0]["metadata"]["labels"]["app.kubernetes.io/managed-by"] | default("") | ternary("False", "True") }}'
+  when: manual_deployment_upgrade is not defined
+
+- name: Determine pod label_selector to be used
+  set_fact:
+    tower_pod_label_selector: '{{ manual_deployment_upgrade|bool | default("false"|bool)| ternary("app={{ deployment_type }}", "app.kubernetes.io/name={{ meta.name }}") }}'
+
+- name: Show help message since deployment was created by a legacy version
+  debug:
+    var: manual_upgrade_msg
+  when: manual_deployment_upgrade|bool
+
+- name: Get the current resource pod information.
+  k8s_info:
+    api_version: v1
+    kind: Pod
+    namespace: '{{ meta.namespace }}'
+    label_selectors:
+      - '{{ tower_pod_label_selector }}'
+    field_selectors:
+      - status.phase=Running
+  register: tower_pods
+  changed_when: tower_pods["resources"] | length
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    tower_pod_name: "{{ tower_pods['resources'][0]['metadata']['name'] | default('') }}"
+
+- name: Apply Resources
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', item + '.yaml.j2') }}"
+    wait: yes
+  register: tower_resources_result
+  loop:
+    - 'tower_config'
+    - 'tower_app_credentials'
+    - 'tower_service_account'
+    - 'tower_persistent'
+    - 'tower_service'
+    - 'tower_ingress'
+  when: not manual_deployment_upgrade|bool
+
+- name: Apply deployment resources
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'tower_deployment.yaml.j2') | from_yaml }}"
+    wait: yes
+  register: tower_deployment_result
+  when: not manual_deployment_upgrade|bool
+
+- block:
+    - name: Delete pod to reload configuration
+      k8s:
+        api_version: v1
+        state: absent
+        kind: Pod
+        namespace: '{{ meta.namespace }}'
+        name: '{{ tower_pod_name }}'
+        wait: yes
+      when:
+        - tower_resources_result.changed
+        - tower_pod_name | length
+
+    - name: Get the new resource pod information after updating resource.
+      k8s_info:
+        kind: Pod
+        namespace: '{{ meta.namespace }}'
+        label_selectors:
+          - '{{ tower_pod_label_selector }}'
+        field_selectors:
+          - status.phase=Running
+      register: _new_pod
+      until:
+        - _new_pod['resources'] | length
+        - _new_pod['resources'][0]['metadata']['name'] != tower_pod_name
+      delay: 5
+      retries: 60
+      when: not manual_deployment_upgrade|bool
+
+    - name: Update new resource pod name as a variable.
+      set_fact:
+        tower_pod_name: '{{ _new_pod["resources"][0]["metadata"]["name"] }}'
+  when:
+    - tower_resources_result.changed or tower_deployment_result.changed
+
+- name: Verify the resource pod name is populated.
+  assert:
+    that: tower_pod_name != ''
+    fail_msg: "Could not find the tower pod's name."

roles/installer/templates/tower_postgres.yaml.j2

@@ -1,6 +1,6 @@
 # Postgres StatefulSet.
 ---
-apiVersion: v1
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: '{{ meta.name }}-postgres'

roles/installer/vars/main.yml

@@ -3,3 +3,4 @@ postgres_initdb_args: '--auth-host=scram-sha-256'
 postgres_host_auth_method: 'scram-sha-256'
 ldap_cacert_ca_crt: ''
 tower_projects_existing_claim: ''
+manual_upgrade_msg: 'Manual Upgrade is required, please refer to the upgrade docs'