set api timeout to match proxy timeout (#2056)

feat: set api timeout to match proxy timeout

Timeout before the openshift route times out
not timing out before undercuts usefulness of our log-traceback-middleware in
django-ansible-base that logs a traceback from requests that get timed
out -- because uwsgi or gunicorn has to send the timeout signal to the
worker handling the request. Also leads to issues where requests that
envoy has already timed out are filling up queues of the workers of the
components.

Also, configure nginx to return a 503 if WSGI server doesn't respond.

Co-authored-by: Elijah DeLee <kdelee@redhat.com>

roles/installer/defaults/main.yml

@@ -489,8 +489,12 @@ ipv6_disabled: false
 #         - hostname
 host_aliases: ''
 
+# receptor default values
 receptor_log_level: info
 
+# common default values
+client_request_timeout: 30
+
 # UWSGI default values
 uwsgi_processes: 5
 # NOTE: to increase this value, net.core.somaxconn must also be increased
@@ -498,6 +502,9 @@ uwsgi_processes: 5
 # Also see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls for how
 # to allow setting this sysctl, which requires kubelet configuration to add to allowlist
 uwsgi_listen_queue_size: 128
+uwsgi_timeout: "{{ (([(client_request_timeout | int), 10] | max) / 3) | int }}"
+uwsgi_timeout_grace_period: 2
+
 
 # NGINX default values
 nginx_worker_processes: 1
@@ -505,6 +512,7 @@ nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
 nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
 nginx_client_max_body_size: 5
+nginx_read_timeout: "{{ (([(client_request_timeout | int), 10] | max) / 2) | int }}" # used in nginx config
 
 extra_settings_files: {}
 

roles/installer/templates/configmaps/config.yaml.j2

@@ -187,7 +187,7 @@ data:
                 allow 127.0.0.1;
                 deny all;
             }
-
+ 
             location {{ (ingress_path + '/static').replace('//', '/') }} {
                 alias /var/lib/awx/public/static/;
             }
@@ -229,7 +229,7 @@ data:
             location {{ ingress_path }} {
                 # Add trailing / if missing
                 rewrite ^(.*)$http_host(.*[^/])$ $1$http_host$2/ permanent;
-                uwsgi_read_timeout 125s;
+                uwsgi_read_timeout {{ nginx_read_timeout }}s;
                 uwsgi_pass uwsgi;
                 include /etc/nginx/uwsgi_params;
                 include /etc/nginx/conf.d/*.conf;
@@ -243,6 +243,23 @@ data:
                 add_header Cache-Control "no-cache, no-store, must-revalidate";
                 add_header Expires "0";
                 add_header Pragma "no-cache";
+                # Return 503 Service Unavailable with JSON response if uWSGI fails to respond
+                error_page 504 =503 /json_503;
+                error_page 502 =503 /json_503;  # Optional, in case uWSGI is completely down   
+            }
+
+            location = /json_503 {
+                # Custom JSON response for 503 Service Unavailable
+                internal;
+                add_header Content-Type application/json;
+
+                # Check if X-Request-ID is set and include it in the response
+                if ($http_x_request_id) {
+                    return 503 '{"status": "error", "message": "Service Unavailable", "code": 503, "request_id": "$http_x_request_id"}';
+                }
+
+                # If X-Request-ID is not set, just return the basic JSON response
+                return 503 '{"status": "error", "message": "Service Unavailable", "code": 503}';
             }
         }
     }
@@ -304,8 +321,8 @@ data:
     max-requests = 1000
     buffer-size = 32768
 
-    harakiri = 120
-    harakiri-graceful-timeout = 115
+    harakiri = {{ uwsgi_timeout }}
+    harakiri-graceful-timeout = {{ uwsgi_timeout_grace_period }}
     harakiri-graceful-signal = 6
     py-call-osafterfork = true