allow configuration of uwsgi timeout

This is important when you have proxies in front of the service
that may have stricter timeouts, as you need at least as strict a
timeout at uwsgi to get meaningful traceback and see source of problem.

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1730,6 +1730,9 @@ spec:
               uwsgi_listen_queue_size:
                 description: Set the socket listen queue size for uwsgi
                 type: integer
+              uwsgi_timeout:
+                description: Set the timeout for requests served by uwsgi. (note, graceful exit signal sent 2 seconds prior to timeout)
+                type: integer
               nginx_worker_processes:
                 description: Set the number of workers for nginx
                 type: integer
@@ -1965,6 +1968,9 @@ spec:
                 description: Disable web container's nginx ipv6 listener
                 type: boolean
                 default: false
+              idle_deployment:
+                description: Scale down deployments to put AWX into an idle state
+                type: boolean
               metrics_utility_enabled:
                 description: Enable metrics utility
                 type: boolean
@@ -2014,6 +2020,7 @@ spec:
                 type: string
             type: object
           status:
+            x-kubernetes-preserve-unknown-fields: true
             properties:
               URL:
                 description: URL to access the deployed instance
@@ -2062,5 +2069,6 @@ spec:
                       type: string
                   type: object
                 type: array
+                x-kubernetes-preserve-unknown-fields: true
             type: object
         type: object

config/manager/manager.yaml

@@ -73,8 +73,8 @@ spec:
             memory: "32Mi"
             cpu: "50m"
           limits:
-            memory: "960Mi"
-            cpu: "1500m"
+            memory: "4000Mi"
+            cpu: "2000m"
       serviceAccountName: controller-manager
       imagePullSecrets:
       - name: redhat-operators-pull-secret

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -554,6 +554,12 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:number
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Uwsgi Timeout
+        path: uwsgi_timeout
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:number
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Uwsgi Processes
         path: uwsgi_processes
         x-descriptors:
@@ -1155,6 +1161,13 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - description: Scale down deployments to put AWX into an idle state
+        displayName: Idle AWX
+        path: idle_deployment
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       statusDescriptors:
       - description: Route to access the instance deployed
         displayName: URL

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -70,15 +70,16 @@ spec:
 
 ## Custom UWSGI Configuration
 
-We allow the customization of two UWSGI parameters:
+We allow the customization of three UWSGI parameters:
 
 * [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5)
 * [listen](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#listen) with `uwsgi_listen_queue_size` (default 128)
+* [harakiri](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#harakiri) with `uwsgi_timeout` (default 30)
 
 **Note:** Increasing the listen queue beyond 128 requires that the sysctl setting net.core.somaxconn be set to an equal value or higher.
   The operator will set the appropriate securityContext sysctl value for you, but it is a required that this sysctl be added to an allowlist on the kubelet level. [See kubernetes docs about allowing this sysctl setting](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls).
 
-These vars relate to the vertical and horizontal scalibility of the web service.
+The `processes` and `listen` vars relate to the vertical and horizontal scalibility of the web service.
 
 Increasing the number of processes allows more requests to be actively handled
 per web pod, but will consume more CPU and Memory and the resource requests
@@ -89,6 +90,12 @@ requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related nginx
 configuration.
 
+The `uwsgi_timeout` variable determines after how many seconds a request will
+be forecibly killed by uwsgi. A "graceful" timeout signal is sent to the worker
+2 seconds prior to attempt to get a traceback of what may be causing the
+request to hang.
+
+
 ## Custom Nginx Configuration
 
 Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf.

roles/installer/defaults/main.yml

@@ -497,6 +497,7 @@ uwsgi_processes: 5
 # Also see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls for how
 # to allow setting this sysctl, which requires kubelet configuration to add to allowlist
 uwsgi_listen_queue_size: 128
+uwsgi_timeout: 30
 
 # NGINX default values
 nginx_worker_processes: 1
@@ -505,3 +506,6 @@ nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
 
 extra_settings_files: {}
+
+# idle_deployment - Scale down deployments to put AWX into an idle state
+idle_deployment: false

roles/installer/tasks/database.yml

@@ -0,0 +1,167 @@
+---
+- name: Get database configuration
+  include_tasks: database_configuration.yml
+
+# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades.
+# So we have to take that into account and preferentially set the most recent one.
+- name: Get the old postgres pod (N-1)
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    field_selectors:
+      - status.phase=Running
+  register: _running_pods
+
+- block:
+    - name: Filter pods by name
+      set_fact:
+        filtered_old_postgres_pods: "{{ _running_pods.resources |
+          selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
+          rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') |
+          list }}"
+
+  # Sort pods by name in reverse order (most recent PG version first) and set
+    - name: Set info for previous postgres pod
+      set_fact:
+        sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
+          sort(attribute='metadata.name') |
+          reverse | list }}"
+      when: filtered_old_postgres_pods | length
+
+
+    - name: Set info for previous postgres pod
+      set_fact:
+        old_postgres_pod: "{{ sorted_old_postgres_pods | first }}"
+      when: filtered_old_postgres_pods | length
+  when: _running_pods.resources | length
+
+- name: Look up details for this deployment
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: this_awx
+
+# If this deployment has been upgraded before or if upgrade has already been started, set this var
+- name: Set previous PG version var
+  set_fact:
+    _previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
+  when:
+    - this_awx['resources'][0] is defined
+    - "'upgradedPostgresVersion' in this_awx['resources'][0]['status']"
+
+- name: Check if postgres pod is running an older version
+  block:
+    - name: Get old PostgreSQL version
+      k8s_exec:
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        pod: "{{ old_postgres_pod['metadata']['name'] }}"
+        command: |
+          bash -c """
+          if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then
+            cat "{{ _postgres_data_path }}/PG_VERSION"
+          elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then
+            cat '/var/lib/postgresql/data/pgdata/PG_VERSION'
+          fi
+          """
+      register: _old_pg_version
+
+    - debug:
+        msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"
+
+    - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
+      include_tasks: upgrade_postgres.yml
+      when:
+        - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
+  when:
+    - managed_database
+    - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version | int < supported_pg_version, true)
+    - old_postgres_pod | length  # If empty, then old pg pod has been removed and we can assume the upgrade is complete
+
+- block:
+    - name: Create Database if no database is specified
+      k8s:
+        apply: true
+        definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}"
+      register: create_statefulset_result
+
+    - name: Scale down Deployment for migration
+      include_tasks: scale_down_deployment.yml
+      when: create_statefulset_result.changed
+
+  rescue:
+    - name: Scale down Deployment for migration
+      include_tasks: scale_down_deployment.yml
+
+    - name: Scale down PostgreSQL statefulset for migration
+      kubernetes.core.k8s_scale:
+        api_version: apps/v1
+        kind: StatefulSet
+        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        replicas: 0
+        wait: yes
+
+    - name: Remove PostgreSQL statefulset for upgrade
+      k8s:
+        state: absent
+        api_version: apps/v1
+        kind: StatefulSet
+        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        wait: yes
+      when: create_statefulset_result.error == 422
+
+    - name: Recreate PostgreSQL statefulset with updated values
+      k8s:
+        apply: true
+        definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}"
+  when: managed_database
+
+- name: Set Default label selector for custom resource generated postgres
+  set_fact:
+    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}"
+  when: postgres_label_selector is not defined
+
+- name: Get the postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+    field_selectors:
+      - status.phase=Running
+  register: postgres_pod
+
+- name: Wait for Database to initialize if managed DB
+  k8s_info:
+    kind: Pod
+    namespace: '{{ ansible_operator_meta.namespace }}'
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+    field_selectors:
+      - status.phase=Running
+  register: postgres_pod
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+    - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
+  delay: 5
+  retries: 60
+  when: managed_database
+
+- name: Look up details for this deployment
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: this_awx
+
+- name: Migrate data from old Openshift instance
+  import_tasks: migrate_data.yml
+  when:
+    - old_pg_config['resources'] is defined
+    - old_pg_config['resources'] | length
+    - this_awx['resources'][0]['status']['migratedFromSecret'] is not defined

roles/installer/tasks/database_configuration.yml

@@ -51,6 +51,14 @@
   set_fact:
     _default_postgres_image: "{{ _postgres_image }}:{{_postgres_image_version }}"
 
+- name: Fail if PostgreSQL secret is specified, but not found
+  fail:
+    msg: "PostgreSQL configuration {{ postgres_configuration_secret }} not found in namespace {{ ansible_operator_meta.namespace }}"
+  when:
+    - postgres_configuration_secret | length
+    - _custom_pg_config_resources is defined
+    - _custom_pg_config_resources['resources'] | length == 0
+
 - name: Set PostgreSQL configuration
   set_fact:
     _pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}'
@@ -106,167 +114,3 @@
 - name: Set database as managed
   set_fact:
     managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"
-
-# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades.
-# So we have to take that into account and preferentially set the most recent one.
-- name: Get the old postgres pod (N-1)
-  k8s_info:
-    kind: Pod
-    namespace: "{{ ansible_operator_meta.namespace }}"
-    field_selectors:
-      - status.phase=Running
-  register: _running_pods
-
-- block:
-    - name: Filter pods by name
-      set_fact:
-        filtered_old_postgres_pods: "{{ _running_pods.resources |
-          selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
-          rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') |
-          list }}"
-
-  # Sort pods by name in reverse order (most recent PG version first) and set
-    - name: Set info for previous postgres pod
-      set_fact:
-        sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
-          sort(attribute='metadata.name') |
-          reverse | list }}"
-      when: filtered_old_postgres_pods | length
-
-
-    - name: Set info for previous postgres pod
-      set_fact:
-        old_postgres_pod: "{{ sorted_old_postgres_pods | first }}"
-      when: filtered_old_postgres_pods | length
-  when: _running_pods.resources | length
-
-- name: Look up details for this deployment
-  k8s_info:
-    api_version: "{{ api_version }}"
-    kind: "{{ kind }}"
-    name: "{{ ansible_operator_meta.name }}"
-    namespace: "{{ ansible_operator_meta.namespace }}"
-  register: this_awx
-
-# If this deployment has been upgraded before or if upgrade has already been started, set this var
-- name: Set previous PG version var
-  set_fact:
-    _previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
-  when:
-    - this_awx['resources'][0] is defined
-    - "'upgradedPostgresVersion' in this_awx['resources'][0]['status']"
-
-- name: Check if postgres pod is running an older version
-  block:
-    - name: Get old PostgreSQL version
-      k8s_exec:
-        namespace: "{{ ansible_operator_meta.namespace }}"
-        pod: "{{ old_postgres_pod['metadata']['name'] }}"
-        command: |
-          bash -c """
-          if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then
-            cat "{{ _postgres_data_path }}/PG_VERSION"
-          elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then
-            cat '/var/lib/postgresql/data/pgdata/PG_VERSION'
-          fi
-          """
-      register: _old_pg_version
-
-    - debug:
-        msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"
-
-    - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
-      include_tasks: upgrade_postgres.yml
-      when:
-        - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
-  when:
-    - managed_database
-    - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version | int < supported_pg_version, true)
-    - old_postgres_pod | length  # If empty, then old pg pod has been removed and we can assume the upgrade is complete
-
-- block:
-    - name: Create Database if no database is specified
-      k8s:
-        apply: true
-        definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}"
-      register: create_statefulset_result
-
-    - name: Scale down Deployment for migration
-      include_tasks: scale_down_deployment.yml
-      when: create_statefulset_result.changed
-
-  rescue:
-    - name: Scale down Deployment for migration
-      include_tasks: scale_down_deployment.yml
-
-    - name: Scale down PostgreSQL statefulset for migration
-      kubernetes.core.k8s_scale:
-        api_version: apps/v1
-        kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
-        namespace: "{{ ansible_operator_meta.namespace }}"
-        replicas: 0
-        wait: yes
-
-    - name: Remove PostgreSQL statefulset for upgrade
-      k8s:
-        state: absent
-        api_version: apps/v1
-        kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
-        namespace: "{{ ansible_operator_meta.namespace }}"
-        wait: yes
-      when: create_statefulset_result.error == 422
-
-    - name: Recreate PostgreSQL statefulset with updated values
-      k8s:
-        apply: true
-        definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}"
-  when: managed_database
-
-- name: Set Default label selector for custom resource generated postgres
-  set_fact:
-    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}"
-  when: postgres_label_selector is not defined
-
-- name: Get the postgres pod information
-  k8s_info:
-    kind: Pod
-    namespace: "{{ ansible_operator_meta.namespace }}"
-    label_selectors:
-      - "{{ postgres_label_selector }}"
-    field_selectors:
-      - status.phase=Running
-  register: postgres_pod
-
-- name: Wait for Database to initialize if managed DB
-  k8s_info:
-    kind: Pod
-    namespace: '{{ ansible_operator_meta.namespace }}'
-    label_selectors:
-      - "{{ postgres_label_selector }}"
-    field_selectors:
-      - status.phase=Running
-  register: postgres_pod
-  until:
-    - "postgres_pod['resources'] | length"
-    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
-    - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
-  delay: 5
-  retries: 60
-  when: managed_database
-
-- name: Look up details for this deployment
-  k8s_info:
-    api_version: "{{ api_version }}"
-    kind: "{{ kind }}"
-    name: "{{ ansible_operator_meta.name }}"
-    namespace: "{{ ansible_operator_meta.namespace }}"
-  register: this_awx
-
-- name: Migrate data from old Openshift instance
-  import_tasks: migrate_data.yml
-  when:
-    - old_pg_config['resources'] is defined
-    - old_pg_config['resources'] | length
-    - this_awx['resources'][0]['status']['migratedFromSecret'] is not defined

roles/installer/tasks/idle_deployment.yml

@@ -0,0 +1,34 @@
+---
+- name: Scale down AWX Deployments
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: "{{ item }}"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      spec:
+        replicas: 0
+  loop:
+    - '{{ ansible_operator_meta.name }}-task'
+    - '{{ ansible_operator_meta.name }}-web'
+
+- name: Get database configuration
+  include_tasks: database_configuration.yml
+
+- name: Scale down PostgreSQL Statefulset
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: apps/v1
+      kind: StatefulSet
+      metadata:
+        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      spec:
+        replicas: 0
+  when: managed_database
+
+- name: End Playbook
+  ansible.builtin.meta: end_play

roles/installer/tasks/install.yml

@@ -74,8 +74,8 @@
 - name: Include set_images tasks
   include_tasks: set_images.yml
 
-- name: Include database configuration tasks
-  include_tasks: database_configuration.yml
+- name: Include Database tasks
+  include_tasks: database.yml
 
 - name: Load Route TLS certificate
   include_tasks: load_route_tls_secret.yml

roles/installer/tasks/main.yml

@@ -1,4 +1,8 @@
 ---
+- name: Idle AWX
+  include_tasks: idle_deployment.yml
+  when: idle_deployment | bool
+
 - name: Check for presence of old awx Deployment
   k8s_info:
     api_version: apps/v1

roles/installer/templates/configmaps/config.yaml.j2

@@ -304,8 +304,8 @@ data:
     max-requests = 1000
     buffer-size = 32768
 
-    harakiri = 120
-    harakiri-graceful-timeout = 115
+    harakiri = {{ uwsgi_timeout|int }}
+    harakiri-graceful-timeout = {{ [(uwsgi_timeout|int - 2), 1] | max }}
     harakiri-graceful-signal = 6
     py-call-osafterfork = true