diff --git a/README.md b/README.md index 6e21b687..07372571 100644 --- a/README.md +++ b/README.md @@ -1090,6 +1090,26 @@ Example spec file for volumes and volume mounts > :warning: **Volume and VolumeMount names cannot contain underscores(_)** +##### Custom UWSGI Configuration +We allow the customization of two UWSGI parameters: + +* [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5) +* [listen](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#listen) with `uwsgi_listen_queue_size` (default 128) + +**Note:** Increasing the listen queue beyond 128 requires that the sysctl setting net.core.somaxconn be set to an equal value or higher. + The operator will set the appropriate securityContext sysctl value for you, but it is a required that this sysctl be added to an allowlist on the kubelet level. [See kubernetes docs about allowing this sysctl setting](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls). + +These vars relate to the vertical and horizontal scalibility of the web service. + +Increasing the number of processes allows more requests to be actively handled +per web pod, but will consume more CPU and Memory and the resource requests +should be increased in tandem. Increasing the listen queue allows uwsgi to +queue up requests not yet being handled by the active worker processes, which +may allow the web pods to handle more "bursty" request patterns if many +requests (more than 128) tend to come in a short period of time, but can all be +handled before any other time outs may apply. Also see related nginx +configuration. + ##### Custom Nginx Configuration Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf. @@ -1100,6 +1120,25 @@ Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it i The AWX nginx config automatically includes /etc/nginx/conf.d/*.conf if present. +Additionally there are some global configuration values in the base nginx +config that are available for setting with individual variables. + +These vars relate to the vertical and horizontal scalibility of the web service. + +Increasing the number of processes allows more requests to be actively handled +per web pod, but will consume more CPU and Memory and the resource requests +should be increased in tandem. Increasing the listen queue allows nginx to +queue up requests not yet being handled by the active worker processes, which +may allow the web pods to handle more "bursty" request patterns if many +requests (more than 128) tend to come in a short period of time, but can all be +handled before any other time outs may apply. Also see related uwsgi +configuration. + +* [worker_processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes) with `nginx_worker_processes` (default of 1) +* [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) with `nginx_worker_cpu_affinity` (default "auto") +* [worker_connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) with `nginx_worker_connections` (minimum of 1024) +* [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size) + ##### Custom Favicon You can use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab. diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 62c408b7..f11f9e21 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -1600,6 +1600,24 @@ spec: web_extra_volume_mounts: description: Specify volume mounts to be added to the Web container type: string + uwsgi_processes: + description: Set the number of uwsgi processes to run in a web container + type: integer + uwsgi_listen_queue_size: + description: Set the socket listen queue size for uwsgi + type: integer + nginx_worker_processes: + description: Set the number of workers for nginx + type: integer + nginx_worker_connections: + description: Set the number of connections per worker for nginx + type: integer + nginx_worker_cpu_affinity: + description: Set the CPU affinity for nginx workers + type: string + nginx_listen_queue_size: + description: Set the socket listen queue size for nginx (defaults to same as uwsgi) + type: integer rsyslog_extra_volume_mounts: description: Specify volume mounts to be added to the Rsyslog container type: string diff --git a/config/manifests/bases/awx-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-operator.clusterserviceversion.yaml index a6c57e59..d071b2fc 100644 --- a/config/manifests/bases/awx-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/awx-operator.clusterserviceversion.yaml @@ -396,6 +396,42 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:number + - displayName: Uwsgi Listen Queue Length + path: uwsgi_listen_queue_size + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Uwsgi Processes + path: uwsgi_processes + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: NGINX Worker Processes + path: nginx_worker_processes + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: NGINX Worker Connections + path: nginx_worker_connections + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: NGINX Worker Process CPU Affinity + path: nginx_worker_cpu_affinity + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:string + - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: NGINX Listen Queue Length + path: nginx_listen_queue_size + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:hidden - displayName: Task Replicas path: task_replicas x-descriptors: diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index f7a541cb..15df2022 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -449,3 +449,17 @@ ipv6_disabled: false host_aliases: '' receptor_log_level: info + +# UWSGI default values +uwsgi_processes: 5 +# NOTE: to increase this value, net.core.somaxconn must also be increased +# see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#setting-sysctls-for-a-pod +# Also see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls for how +# to allow setting this sysctl, which requires kubelet configuration to add to allowlist +uwsgi_listen_queue_size: 128 + +# NGINX default values +nginx_worker_processes: 1 +nginx_worker_connections: "{{ uwsgi_listen_queue_size }}" +nginx_worker_cpu_affinity: 'auto' +nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}" diff --git a/roles/installer/templates/configmaps/config.yaml.j2 b/roles/installer/templates/configmaps/config.yaml.j2 index 4be3c2dc..649f3dd4 100644 --- a/roles/installer/templates/configmaps/config.yaml.j2 +++ b/roles/installer/templates/configmaps/config.yaml.j2 @@ -92,11 +92,12 @@ data: {% endfor %} nginx_conf: | - worker_processes 1; + worker_processes {{ nginx_worker_processes }}; + worker_cpu_affinity {{ nginx_worker_cpu_affinity }}; pid /tmp/nginx.pid; events { - worker_connections 1024; + worker_connections {% if nginx_worker_connections > 1024 %}{{ nginx_worker_connections }}{% else %}1024{% endif %}; } http { @@ -131,9 +132,9 @@ data: {% if route_tls_termination_mechanism | lower == 'passthrough' %} server { - listen 8052 default_server; + listen 8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %}; {% if not ipv6_disabled %} - listen [::]:8052 default_server; + listen [::]:8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %}; {% endif %} server_name _; @@ -144,9 +145,9 @@ data: server { {% if route_tls_termination_mechanism | lower == 'passthrough' %} - listen 8053 ssl; + listen 8053 ssl{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %}; {% if not ipv6_disabled %} - listen [::]:8053 ssl; + listen [::]:8053 ssl{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %}; {% endif %} ssl_certificate /etc/nginx/pki/web.crt; @@ -157,9 +158,9 @@ data: ssl_ciphers PROFILE=SYSTEM; ssl_prefer_server_ciphers on; {% else %} - listen 8052 default_server; + listen 8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %}; {% if not ipv6_disabled %} - listen [::]:8052 default_server; + listen [::]:8052 default_server{% if nginx_listen_queue_size > 511 %} backlog={{ nginx_listen_queue_size }}{% endif %}; {% endif %} {% endif %} @@ -276,3 +277,24 @@ data: - work-signing: privatekey: /etc/receptor/work_private_key.pem tokenexpiration: 1m + uwsgi_conf: | + [uwsgi] + socket = 127.0.0.1:8050 + processes = {{ uwsgi_processes }} + listen = {{ uwsgi_listen_queue_size }} + master = true + vacuum = true + no-orphans = true + lazy-apps = true + manage-script-name = true + master-fifo = /var/lib/awx/awxfifo + max-requests = 1000 + buffer-size = 32768 + + if-env = UWSGI_MOUNT_PATH + mount = %(_)=awx.wsgi:application + endif = + + if-not-env = UWSGI_MOUNT_PATH + mount = /=awx.wsgi:application + endif = diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 26ce74cc..03efbe55 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -49,6 +49,12 @@ spec: {{ annotations | indent(width=8) }} {% endif %} spec: +{% if uwsgi_listen_queue_size is defined and uwsgi_listen_queue_size|int > 128 %} + securityContext: + sysctls: + - name: net.core.somaxconn + value: "{{ uwsgi_listen_queue_size }}" +{% endif %} serviceAccountName: '{{ ansible_operator_meta.name }}' {% if image_pull_secret is defined %} imagePullSecrets: @@ -165,6 +171,10 @@ spec: subPath: bundle-ca.crt readOnly: true {% endif %} + - name: {{ ansible_operator_meta.name }}-uwsgi-config + mountPath: "/etc/tower/uwsgi.ini" + subPath: uwsgi.conf + readOnly: true - name: "{{ ansible_operator_meta.name }}-application-credentials" mountPath: "/etc/tower/conf.d/execution_environments.py" subPath: execution_environments.py @@ -399,6 +409,12 @@ spec: items: - key: redis_conf path: redis.conf + - name: {{ ansible_operator_meta.name }}-uwsgi-config + configMap: + name: {{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap + items: + - key: uwsgi_conf + path: uwsgi.conf - name: {{ ansible_operator_meta.name }}-redis-socket emptyDir: {} - name: {{ ansible_operator_meta.name }}-redis-data