From 4c5429190c9342a25b78af26b81366d218dd4df2 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Tue, 29 Aug 2023 15:36:48 -0400
Subject: [PATCH] Timeout stream keep alive for Upgrades and Restores (#1542)

Signed-off-by: Christian M. Adams <chadams@redhat.com>
---
 roles/installer/tasks/migrate_data.yml     |  2 +-
 roles/installer/tasks/upgrade_postgres.yml | 22 +++++++++++++++++++---
 roles/restore/tasks/postgres.yml           | 20 ++++++++++++++++++--
 3 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/roles/installer/tasks/migrate_data.yml b/roles/installer/tasks/migrate_data.yml
index ae103561..3954d48c 100644
--- a/roles/installer/tasks/migrate_data.yml
+++ b/roles/installer/tasks/migrate_data.yml
@@ -76,7 +76,7 @@
       trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
       echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD=\"$PGPASSWORD_OLD\" {{ pgdump }} |  PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      PGPASSWORD=\"$PGPASSWORD_OLD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
       set +e +o pipefail
       echo 'Successful'
       "
diff --git a/roles/installer/tasks/upgrade_postgres.yml b/roles/installer/tasks/upgrade_postgres.yml
index 5edae076..5c9afc86 100644
--- a/roles/installer/tasks/upgrade_postgres.yml
+++ b/roles/installer/tasks/upgrade_postgres.yml
@@ -91,11 +91,27 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
     pod: "{{ postgres_pod_name }}"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data to new PostgreSQL {{ supported_postgres_version }} Database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
-      PGPASSWORD='{{ awx_postgres_pass }}' {{ pgdump }} | PGPASSWORD='{{ awx_postgres_pass }}' {{ pg_restore }}
+      PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pgdump }} | PGPASSWORD=\"$POSTGRES_PASSWORD\" {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   no_log: "{{ no_log }}"
   register: data_migration
   failed_when: "'Successful' not in data_migration.stdout"
diff --git a/roles/restore/tasks/postgres.yml b/roles/restore/tasks/postgres.yml
index 7f0753ec..78e3ad99 100644
--- a/roles/restore/tasks/postgres.yml
+++ b/roles/restore/tasks/postgres.yml
@@ -87,11 +87,27 @@
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Migrating data from old database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
       cat {{ backup_dir }}/tower.db | PGPASSWORD='{{ awx_postgres_pass }}' {{ pg_restore }}
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
   failed_when: "'Successful' not in data_migration.stdout"