From 6a2b42cdde8a4a5591bbd112e01772fb84aa5fa6 Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Wed, 19 Feb 2025 14:38:54 -0500 Subject: [PATCH] allow configuration of uwsgi timeout This is important when you have proxies in front of the service that may have stricter timeouts, as you need at least as strict a timeout at uwsgi to get meaningful traceback and see source of problem. --- config/crd/bases/awx.ansible.com_awxs.yaml | 3 +++ .../bases/awx-operator.clusterserviceversion.yaml | 6 ++++++ .../custom-volume-and-volume-mount-options.md | 11 +++++++++-- roles/installer/defaults/main.yml | 1 + roles/installer/templates/configmaps/config.yaml.j2 | 4 ++-- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index a3fb9dfa..6f160d2f 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -1730,6 +1730,9 @@ spec: uwsgi_listen_queue_size: description: Set the socket listen queue size for uwsgi type: integer + uwsgi_timeout: + description: Set the timeout for requests served by uwsgi. (note, graceful exit signal sent 2 seconds prior to timeout) + type: integer nginx_worker_processes: description: Set the number of workers for nginx type: integer diff --git a/config/manifests/bases/awx-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-operator.clusterserviceversion.yaml index f5ee7a7b..060b6960 100644 --- a/config/manifests/bases/awx-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/awx-operator.clusterserviceversion.yaml @@ -554,6 +554,12 @@ spec: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:number - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Uwsgi Timeout + path: uwsgi_timeout + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:hidden - displayName: Uwsgi Processes path: uwsgi_processes x-descriptors: diff --git a/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md b/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md index d3f2a99b..de119c65 100644 --- a/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md +++ b/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md @@ -70,15 +70,16 @@ spec: ## Custom UWSGI Configuration -We allow the customization of two UWSGI parameters: +We allow the customization of three UWSGI parameters: * [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5) * [listen](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#listen) with `uwsgi_listen_queue_size` (default 128) +* [harakiri](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#harakiri) with `uwsgi_timeout` (default 30) **Note:** Increasing the listen queue beyond 128 requires that the sysctl setting net.core.somaxconn be set to an equal value or higher. The operator will set the appropriate securityContext sysctl value for you, but it is a required that this sysctl be added to an allowlist on the kubelet level. [See kubernetes docs about allowing this sysctl setting](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls). -These vars relate to the vertical and horizontal scalibility of the web service. +The `processes` and `listen` vars relate to the vertical and horizontal scalibility of the web service. Increasing the number of processes allows more requests to be actively handled per web pod, but will consume more CPU and Memory and the resource requests @@ -89,6 +90,12 @@ requests (more than 128) tend to come in a short period of time, but can all be handled before any other time outs may apply. Also see related nginx configuration. +The `uwsgi_timeout` variable determines after how many seconds a request will +be forecibly killed by uwsgi. A "graceful" timeout signal is sent to the worker +2 seconds prior to attempt to get a traceback of what may be causing the +request to hang. + + ## Custom Nginx Configuration Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf. diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index e9933efa..e71dca69 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -497,6 +497,7 @@ uwsgi_processes: 5 # Also see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls for how # to allow setting this sysctl, which requires kubelet configuration to add to allowlist uwsgi_listen_queue_size: 128 +uwsgi_timeout: 30 # NGINX default values nginx_worker_processes: 1 diff --git a/roles/installer/templates/configmaps/config.yaml.j2 b/roles/installer/templates/configmaps/config.yaml.j2 index 99acf723..eda706a1 100644 --- a/roles/installer/templates/configmaps/config.yaml.j2 +++ b/roles/installer/templates/configmaps/config.yaml.j2 @@ -304,8 +304,8 @@ data: max-requests = 1000 buffer-size = 32768 - harakiri = 120 - harakiri-graceful-timeout = 115 + harakiri = {{ uwsgi_timeout|int }} + harakiri-graceful-timeout = {{ [(uwsgi_timeout|int - 2), 1] | max }} harakiri-graceful-signal = 6 py-call-osafterfork = true