Ensure that web and task deployments scale down for upgrades (#1522)

Co-authored-by: Imed Aouidene <imaouide@imaouide-thinkpadt14sgen2i.cdg.csb>

.github/workflows/ci.yaml

@@ -18,9 +18,9 @@ jobs:
     env:
       DOCKER_API_VERSION: "1.41"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
@@ -45,12 +45,12 @@ jobs:
     runs-on: ubuntu-latest
     name: helm
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.8.0
 
       - name: Build operator image and load into kind
         run: |
@@ -88,7 +88,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Check no_log statements
         run: |

.github/workflows/devel.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Push devel image
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Build Image
         run: |

.github/workflows/label_issue.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Label Issue - Community
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
       - name: Install python requests
         run: pip install requests

.github/workflows/label_pr.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Label PR - Community
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
       - name: Install python requests
         run: pip install requests

.github/workflows/stage.yml

@@ -38,13 +38,13 @@ jobs:
           exit 0
 
       - name: Checkout awx
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ github.repository_owner }}/awx
           path: awx
 
       - name: Checkout awx-operator
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ github.repository_owner }}/awx-operator
           path: awx-operator

README.md

@@ -47,7 +47,7 @@ All of our usage and configuration docs are nested in the `docs/` directory. Bel
   - [Authors](./docs/contributors-guide/author.md)
 - Installation
   - [Basic Install](./docs/installation/basic-install.md)
-  - [Creating a Minikube cluster for testing](./docs/creating-a-minikube-cluster-for-testing.md)
+  - [Creating a Minikube cluster for testing](./docs/installation/creating-a-minikube-cluster-for-testing.md)
   - [Helm Install](./docs/installation/helm-install-on-existing-cluster.md)
 - [Migration](./docs/migration/migration.md)
 - [Uninstall](./docs/uninstall/uninstall.md)
@@ -56,7 +56,7 @@ All of our usage and configuration docs are nested in the `docs/` directory. Bel
   - [Database Configuration](./docs/user-guide/database-configuration.md)
   - [Network and TLS Configuration](./docs/user-guide/network-and-tls-configuration.md)
   - Advanced Configuration
-    - [No Log](./docs/no-log.md)
+    - [No Log](./docs/user-guide/advanced-configuration/no-log.md)
     - [Deploy a Specific Version of AWX](./docs/user-guide/advanced-configuration/deploying-a-specific-version-of-awx.md)
     - [Resource Requirements](./docs/user-guide/advanced-configuration/containers-resource-requirements.md)
     - [Extra Settings](./docs/user-guide/advanced-configuration/extra-settings.md)

molecule/default/tasks/awx_replicas_test.yml

@@ -1,64 +1,64 @@
 ---
 - block:
-  - debug:
-      msg: test - web_replicas and task_replicas should override replicas
+    - debug:
+        msg: test - web_replicas and task_replicas should override replicas
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 2
-        web_replicas: 0
-        task_replicas: 0
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 2
+          web_replicas: 0
+          task_replicas: 0
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 0
-      expected_task_replicas: 0
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 0
+        expected_task_replicas: 0
 
 ####
 
-  - debug:
-      msg: test - replicas should act as a default
+    - debug:
+        msg: test - replicas should act as a default
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 2
-        web_replicas: 1
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 2
+          web_replicas: 1
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 1
-      expected_task_replicas: 2
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 1
+        expected_task_replicas: 2
 
 ####
 
-  - debug:
-      msg: test - replicas=0 should kill all pods
+    - debug:
+        msg: test - replicas=0 should kill all pods
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 0
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 0
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 0
-      expected_task_replicas: 0
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 0
+        expected_task_replicas: 0
 
 ####
 
-  - debug:
-      msg: test - replicas=3 should give 3 of each
+    - debug:
+        msg: test - replicas=3 should give 3 of each
 
-  - include_tasks: apply_awx_spec.yml
-    vars:
-      additional_fields:
-        replicas: 3
+    - include_tasks: apply_awx_spec.yml
+      vars:
+        additional_fields:
+          replicas: 3
 
-  - include_tasks: _test_case_replicas.yml
-    vars:
-      expected_web_replicas: 3
-      expected_task_replicas: 3
+    - include_tasks: _test_case_replicas.yml
+      vars:
+        expected_web_replicas: 3
+        expected_task_replicas: 3
   tags:
     - replicas

roles/installer/tasks/migrate_data.yml

@@ -76,7 +76,7 @@
       trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
       echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD=\"$PGPASSWORD_OLD\" {{ pgdump }} |  PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      PGPASSWORD=\"$PGPASSWORD_OLD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
       set +e +o pipefail
       echo 'Successful'
       "

roles/installer/tasks/scale_down_deployment.yml

@@ -1,12 +1,14 @@
 ---
-
 - name: Check for presence of Deployment
   k8s_info:
     api_version: apps/v1
     kind: Deployment
-    name: "{{ ansible_operator_meta.name }}"
     namespace: "{{ ansible_operator_meta.namespace }}"
-  register: this_deployment
+    label_selectors:
+      - 'app.kubernetes.io/part-of={{ ansible_operator_meta.name }}'
+      - 'app.kubernetes.io/managed-by={{ deployment_type }}-operator'
+      - 'app.kubernetes.io/component={{ deployment_type }}'
+  register: _deployments
 
 - name: Scale down Deployment for migration
   kubernetes.core.k8s_scale:
@@ -16,8 +18,5 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     replicas: 0
     wait: yes
-    wait_timeout: "{{ termination_grace_period_seconds | default(120) }}"
-  loop:
-    - "{{ ansible_operator_meta.name }}-task"
-    - "{{ ansible_operator_meta.name }}-web"
-  when: this_deployment['resources'] | length
+  loop: "{{ _deployments.resources | map(attribute='metadata.name') | list }}"
+  when: _deployments.resources | length

roles/installer/tasks/upgrade_postgres.yml

@@ -91,11 +91,27 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     pod: "{{ postgres_pod_name }}"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data to new PostgreSQL {{ supported_postgres_version }} Database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD='{{ awx_postgres_pass }}' {{ pgdump }} | PGPASSWORD='{{ awx_postgres_pass }}' {{ pg_restore }}
+      PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   no_log: "{{ no_log }}"
   register: data_migration
   failed_when: "'Successful' not in data_migration.stdout"

roles/restore/tasks/postgres.yml

@@ -50,7 +50,7 @@
   k8s_info:
     api_version: apps/v1
     kind: Deployment
-    name: "{{ ansible_operator_meta.namespace }}-task"
+    name: "{{ deployment_name }}-task"
     namespace: "{{ ansible_operator_meta.namespace }}"
   register: this_deployment
 
@@ -63,8 +63,8 @@
     replicas: 0
     wait: yes
   loop:
-    - "{{ ansible_operator_meta.name }}-task"
-    - "{{ ansible_operator_meta.name }}-web"
+    - "{{ deployment_name }}-task"
+    - "{{ deployment_name }}-web"
   when: this_deployment['resources'] | length
 
 - name: Set full resolvable host name for postgres pod
@@ -87,11 +87,27 @@
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data from old database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
       cat {{ backup_dir }}/tower.db | PGPASSWORD='{{ awx_postgres_pass }}' {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
   failed_when: "'Successful' not in data_migration.stdout"