add uwsgi config as configmap with vars (#1487)

README.md

@@ -1090,6 +1090,26 @@ Example spec file for volumes and volume mounts
 
 > :warning: **Volume and VolumeMount names cannot contain underscores(_)**
 
+##### Custom UWSGI Configuration
+We allow the customization of two UWSGI parameters:
+
+* [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5)
+* [listen](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#listen) with `uwsgi_listen_queue_size` (default 128)
+
+**Note:** Increasing the listen queue beyond 128 requires that the sysctl setting net.core.somaxconn be set to an equal value or higher.
+  The operator will set the appropriate securityContext sysctl value for you, but it is a required that this sysctl be added to an allowlist on the kubelet level. [See kubernetes docs about allowing this sysctl setting](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls).
+
+These vars relate to the vertical and horizontal scalibility of the web service.
+
+Increasing the number of processes allows more requests to be actively handled
+per web pod, but will consume more CPU and Memory and the resource requests
+should be increased in tandem.  Increasing the listen queue allows uwsgi to
+queue up requests not yet being handled by the active worker processes, which
+may allow the web pods to handle more "bursty" request patterns if many
+requests (more than 128) tend to come in a short period of time, but can all be
+handled before any other time outs may apply. Also see related nginx
+configuration.
+
 ##### Custom Nginx Configuration
 
 Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf.
@@ -1100,6 +1120,25 @@ Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it i
 
 The AWX nginx config automatically includes /etc/nginx/conf.d/*.conf if present.
 
+Additionally there are some global configuration values in the base nginx
+config that are available for setting with individual variables.
+
+These vars relate to the vertical and horizontal scalibility of the web service.
+
+Increasing the number of processes allows more requests to be actively handled
+per web pod, but will consume more CPU and Memory and the resource requests
+should be increased in tandem.  Increasing the listen queue allows nginx to
+queue up requests not yet being handled by the active worker processes, which
+may allow the web pods to handle more "bursty" request patterns if many
+requests (more than 128) tend to come in a short period of time, but can all be
+handled before any other time outs may apply. Also see related uwsgi
+configuration.
+
+* [worker_processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes) with `nginx_worker_processes` (default of 1)
+* [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) with `nginx_worker_cpu_affinity` (default "auto")
+* [worker_connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) with `nginx_worker_connections` (minimum of 1024)
+* [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size)
+
 ##### Custom Favicon
 
 You can use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -1600,6 +1600,24 @@ spec:
               web_extra_volume_mounts:
                 description: Specify volume mounts to be added to the Web container
                 type: string
+              uwsgi_processes:
+                description: Set the number of uwsgi processes to run in a web container
+                type: integer
+              uwsgi_listen_queue_size:
+                description: Set the socket listen queue size for uwsgi
+                type: integer
+              nginx_worker_processes:
+                description: Set the number of workers for nginx
+                type: integer
+              nginx_worker_connections:
+                description: Set the number of connections per worker for nginx
+                type: integer
+              nginx_worker_cpu_affinity:
+                description: Set the CPU affinity for nginx workers
+                type: string
+              nginx_listen_queue_size:
+                description: Set the socket listen queue size for nginx (defaults to same as uwsgi)
+                type: integer
               rsyslog_extra_volume_mounts:
                 description: Specify volume mounts to be added to the Rsyslog container
                 type: string

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -396,6 +396,42 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:number
+      - displayName: Uwsgi Listen Queue Length
+        path: uwsgi_listen_queue_size
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:number
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Uwsgi Processes
+        path: uwsgi_processes
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:number
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: NGINX Worker Processes
+        path: nginx_worker_processes
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:number
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: NGINX Worker Connections
+        path: nginx_worker_connections
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:number
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: NGINX Worker Process CPU Affinity
+        path: nginx_worker_cpu_affinity
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:string
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: NGINX Listen Queue Length
+        path: nginx_listen_queue_size
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:number
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Task Replicas
         path: task_replicas
         x-descriptors:

roles/installer/defaults/main.yml

@@ -449,3 +449,17 @@ ipv6_disabled: false
 host_aliases: ''
 
 receptor_log_level: info
+
+# UWSGI default values
+uwsgi_processes: 5
+# NOTE: to increase this value, net.core.somaxconn must also be increased
+# see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#setting-sysctls-for-a-pod
+# Also see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls for how
+# to allow setting this sysctl, which requires kubelet configuration to add to allowlist
+uwsgi_listen_queue_size: 128
+
+# NGINX default values
+nginx_worker_processes: 1
+nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
+nginx_worker_cpu_affinity: 'auto'
+nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"

roles/installer/templates/configmaps/config.yaml.j2

@@ -92,11 +92,12 @@ data:
 {% endfor %}
 
   nginx_conf: |
-    worker_processes  1;
+    worker_processes  {{ nginx_worker_processes }};
+    worker_cpu_affinity  {{ nginx_worker_cpu_affinity }};
     pid        /tmp/nginx.pid;
 
     events {
-        worker_connections  1024;
+        worker_connections {% if nginx_worker_connections > 1024 %}{{ nginx_worker_connections }}{% else %}1024{% endif %};
     }
 
     http {
@@ -131,9 +132,9 @@ data:
 
         {% if route_tls_termination_mechanism | lower == 'passthrough' %}
         server {
-            listen 8052 default_server;
+            listen 8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %};
             {% if not ipv6_disabled %}
-            listen [::]:8052 default_server;
+            listen [::]:8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %};
             {% endif %}
             server_name _;
 
@@ -144,9 +145,9 @@ data:
 
         server {
         {% if route_tls_termination_mechanism | lower == 'passthrough' %}
-            listen 8053 ssl;
+            listen 8053 ssl{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %};
             {% if not ipv6_disabled %}
-            listen [::]:8053 ssl;
+            listen [::]:8053 ssl{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %};
             {% endif %}
 
             ssl_certificate /etc/nginx/pki/web.crt;
@@ -157,9 +158,9 @@ data:
             ssl_ciphers PROFILE=SYSTEM;
             ssl_prefer_server_ciphers on;
         {% else %}
-            listen 8052 default_server;
+            listen 8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %};
             {% if not ipv6_disabled %}
-            listen [::]:8052 default_server;
+            listen [::]:8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %};
             {% endif %}
         {% endif %}
 
@@ -276,3 +277,24 @@ data:
     - work-signing:
         privatekey: /etc/receptor/work_private_key.pem
         tokenexpiration: 1m
+  uwsgi_conf: |
+    [uwsgi]
+    socket = 127.0.0.1:8050
+    processes = {{ uwsgi_processes }}
+    listen = {{ uwsgi_listen_queue_size }}
+    master = true
+    vacuum = true
+    no-orphans = true
+    lazy-apps = true
+    manage-script-name = true
+    master-fifo = /var/lib/awx/awxfifo
+    max-requests = 1000
+    buffer-size = 32768
+    
+    if-env = UWSGI_MOUNT_PATH
+    mount = %(_)=awx.wsgi:application
+    endif =
+    
+    if-not-env = UWSGI_MOUNT_PATH
+    mount = /=awx.wsgi:application
+    endif =

roles/installer/templates/deployments/web.yaml.j2

@@ -49,6 +49,12 @@ spec:
         {{ annotations | indent(width=8) }}
 {% endif %}
     spec:
+{% if uwsgi_listen_queue_size is defined and uwsgi_listen_queue_size|int > 128 %}
+      securityContext:
+        sysctls:
+        - name: net.core.somaxconn
+          value: "{{ uwsgi_listen_queue_size }}"
+{% endif %}
       serviceAccountName: '{{ ansible_operator_meta.name }}'
 {% if image_pull_secret is defined %}
       imagePullSecrets:
@@ -165,6 +171,10 @@ spec:
               subPath: bundle-ca.crt
               readOnly: true
 {% endif %}
+            - name: {{ ansible_operator_meta.name }}-uwsgi-config
+              mountPath: "/etc/tower/uwsgi.ini"
+              subPath: uwsgi.conf
+              readOnly: true
             - name: "{{ ansible_operator_meta.name }}-application-credentials"
               mountPath: "/etc/tower/conf.d/execution_environments.py"
               subPath: execution_environments.py
@@ -399,6 +409,12 @@ spec:
             items:
               - key: redis_conf
                 path: redis.conf
+        - name: {{ ansible_operator_meta.name }}-uwsgi-config
+          configMap:
+            name: {{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap
+            items:
+              - key: uwsgi_conf
+                path: uwsgi.conf
         - name: {{ ansible_operator_meta.name }}-redis-socket
           emptyDir: {}
         - name: {{ ansible_operator_meta.name }}-redis-data