[web/task split] split web and task deployment + a few supporting bits (#1189)

* first pass, still WIP, need tolerations etc * add tolerations that don't work bc idk * bug hunting * local push, still a WIP * affinity still needs testfor to_nice_yaml, tolerations logic is working * fixed task deployment and affinity for both
2026-05-06 05:12:47 +00:00 · 2023-02-06 16:08:35 -05:00
parent ebc040fe75
commit d9f3a428d4
5 changed files with 1265 additions and 22 deletions
--- a/config/crd/bases/awx.ansible.com_awxs.yaml
+++ b/config/crd/bases/awx.ansible.com_awxs.yaml
--- a/roles/installer/defaults/main.yml
+++ b/roles/installer/defaults/main.yml
@@ -93,6 +93,58 @@ affinity: {}
 #     effect: "NoSchedule"
 tolerations: ''

+# Add node tolerations for the task pods. Specify as literal block. E.g.:
+# task_tolerations: |
+#   - key: "dedicated"
+#     operator: "Equal"
+#     value: "AWXtask"
+#     effect: "NoSchedule"
+task_tolerations: ''
+
+# Add node tolerations for the web pods. Specify as literal block. E.g.:
+# web_tolerations: |
+#   - key: "dedicated"
+#     operator: "Equal"
+#     value: "AWXweb"
+#     effect: "NoSchedule"
+web_tolerations: ''
+
+# Add affinities for all pods
+#  affinity:
+#    nodeAffinity:
+#      requiredDuringSchedulingIgnoredDuringExecution:
+#        nodeSelectorTerms:
+#        - matchExpressions:
+#          - key: app.kubernetes.io/component
+#            operator: In
+#            values:
+#            - awx
+affinity: {}
+
+# Add affinities for all task pods
+#  affinity:
+#    nodeAffinity:
+#      requiredDuringSchedulingIgnoredDuringExecution:
+#        nodeSelectorTerms:
+#        - matchExpressions:
+#          - key: app.kubernetes.io/name
+#            operator: In
+#            values:
+#            - awx-task
+task_affinity: {}
+
+# Add affinities for all web pods
+#  affinity:
+#    nodeAffinity:
+#      requiredDuringSchedulingIgnoredDuringExecution:
+#        nodeSelectorTerms:
+#        - matchExpressions:
+#          - key: app.kubernetes.io/name
+#            operator: In
+#            values:
+#            - awx-web
+web_affinity: {}
+
 # Add annotations to awx pods. Specify as literal block. E.g.:
 # annotations: |
 #   my.annotation/1: value
@@ -181,6 +233,9 @@ task_command: []
 web_args:
  - /usr/bin/launch_awx.sh
 web_command: []
+ryslog_args:
+  - /usr/bin/launch_awx_rsyslog.sh
+rsyslog_command: []

 task_resource_requirements:
  requests:
@@ -197,6 +252,12 @@ ee_resource_requirements:
    cpu: 100m
    memory: 64Mi

+# TODO: validate default resource requirements
+rsyslog_resource_requirements:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+
 # Customize CSRF options
 csrf_cookie_secure: False
 session_cookie_secure: False
--- a/roles/installer/tasks/resources_configuration.yml
+++ b/roles/installer/tasks/resources_configuration.yml
@@ -6,7 +6,7 @@
    kind: Pod
    namespace: '{{ ansible_operator_meta.namespace }}'
    label_selectors:
-      - "app.kubernetes.io/name={{ ansible_operator_meta.name }}"
+      - "app.kubernetes.io/name={{ ansible_operator_meta.name }}-task"
      - "app.kubernetes.io/managed-by={{ deployment_type }}-operator"
      - "app.kubernetes.io/component={{ deployment_type }}"
    field_selectors:
--- a/roles/installer/templates/deployments/task.yaml.j2
+++ b/roles/installer/templates/deployments/task.yaml.j2
@@ -201,12 +201,8 @@ spec:
              readOnly: true
            - name: {{ ansible_operator_meta.name }}-redis-socket
              mountPath: "/var/run/redis"
-            - name: supervisor-socket
-              mountPath: "/var/run/supervisor"
            - name: rsyslog-socket
              mountPath: "/var/run/awx-rsyslog"
-            - name: rsyslog-dir
-              mountPath: "/var/lib/awx/rsyslog"
            - name: "{{ ansible_operator_meta.name }}-receptor-config"
              mountPath: "/etc/receptor/"
            - name: "{{ ansible_operator_meta.name }}-receptor-work-signing"
@@ -326,6 +322,43 @@ spec:
 {% endif %}
 {% if ee_extra_env -%}
            {{ ee_extra_env | indent(width=12, first=True) }}
+{% endif %}
+        - image: '{{ _image }}'
+          name: '{{ ansible_operator_meta.name }}-rsyslog'
+{% if rsyslog_command %}
+          command: {{ rsyslog_command }}
+{% endif %}
+{% if ryslog_args %}
+          args: {{ ryslog_args }}
+{% endif %}
+          imagePullPolicy: '{{ image_pull_policy }}'
+          volumeMounts:
+            - name: "{{ ansible_operator_meta.name }}-application-credentials"
+              mountPath: "/etc/tower/conf.d/credentials.py"
+              subPath: credentials.py
+              readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
+            - name: {{ ansible_operator_meta.name }}-settings
+              mountPath: "/etc/tower/settings.py"
+              subPath: settings.py
+              readOnly: true
+            - name: {{ ansible_operator_meta.name }}-redis-socket
+              mountPath: "/var/run/redis"
+            - name: rsyslog-socket
+              mountPath: "/var/run/awx-rsyslog"
+{% if development_mode | bool %}
+            - name: awx-devel
+              mountPath: "/awx_devel"
+{% endif %}
+          env:
+            - name: SUPERVISOR_WEB_CONFIG_PATH
+              value: "/etc/supervisor_rsyslog.conf"
+{% if development_mode | bool %}
+            - name: AWX_KUBE_DEVEL
+              value: "1"
 {% endif %}
 {% if node_selector %}
      nodeSelector:
@@ -335,14 +368,20 @@ spec:
      topologySpreadConstraints:
        {{ topology_spread_constraints | indent(width=8) }}
 {% endif %}
-{% if affinity | length %}
+{% if task_tolerations %}
+      tolerations:
+        {{ task_tolerations | to_nice_yaml | indent(width=8) }}
+{% elif tolerations %}
+      tolerations:
+        {{ tolerations | to_nice_yaml | indent(width=8) }}
+{% endif %}
+{% if task_affinity %}
+      affinity:
+        {{ task_affinity | to_nice_yaml | indent(width=8) }}
+{% elif affinity %}
      affinity:
        {{ affinity | to_nice_yaml | indent(width=8) }}
 {% endif %}
-{% if tolerations %}
-      tolerations:
-        {{ tolerations | indent(width=8) }}
-{% endif %}
 {% if (projects_persistence|bool and is_k8s|bool) or (security_context_settings|length) %}
      securityContext:
 {% if projects_persistence|bool and is_k8s|bool %}
@@ -438,14 +477,10 @@ spec:
          emptyDir: {}
        - name: {{ ansible_operator_meta.name }}-redis-data
          emptyDir: {}
-        - name: supervisor-socket
-          emptyDir: {}
        - name: rsyslog-socket
          emptyDir: {}
        - name: receptor-socket
          emptyDir: {}
-        - name: rsyslog-dir
-          emptyDir: {}
        - name: {{ ansible_operator_meta.name }}-receptor-config
          emptyDir: {}
        - name: {{ ansible_operator_meta.name }}-default-receptor-config
--- a/roles/installer/templates/deployments/web.yaml.j2
+++ b/roles/installer/templates/deployments/web.yaml.j2
@@ -156,12 +156,8 @@ spec:
              readOnly: true
            - name: {{ ansible_operator_meta.name }}-redis-socket
              mountPath: "/var/run/redis"
-            - name: supervisor-socket
-              mountPath: "/var/run/supervisor"
            - name: rsyslog-socket
              mountPath: "/var/run/awx-rsyslog"
-            - name: rsyslog-dir
-              mountPath: "/var/lib/awx/rsyslog"
            - name: "{{ ansible_operator_meta.name }}-projects"
              mountPath: "/var/lib/awx/projects"
 {% if development_mode | bool %}
@@ -190,6 +186,57 @@ spec:
            {{ web_extra_env | indent(width=12, first=True) }}
 {% endif %}
          resources: {{ web_resource_requirements }}
+        - image: '{{ _image }}'
+          name: '{{ ansible_operator_meta.name }}-rsyslog'
+{% if rsyslog_command %}
+          command: {{ rsyslog_command }}
+{% endif %}
+{% if ryslog_args %}
+          args: {{ ryslog_args }}
+{% endif %}
+          imagePullPolicy: '{{ image_pull_policy }}'
+          volumeMounts:
+            - name: "{{ ansible_operator_meta.name }}-application-credentials"
+              mountPath: "/etc/tower/conf.d/credentials.py"
+              subPath: credentials.py
+              readOnly: true
+            - name: "{{ secret_key_secret_name }}"
+              mountPath: /etc/tower/SECRET_KEY
+              subPath: SECRET_KEY
+              readOnly: true
+            - name: {{ ansible_operator_meta.name }}-settings
+              mountPath: "/etc/tower/settings.py"
+              subPath: settings.py
+              readOnly: true
+            - name: {{ ansible_operator_meta.name }}-redis-socket
+              mountPath: "/var/run/redis"
+            - name: rsyslog-socket
+              mountPath: "/var/run/awx-rsyslog"
+{% if development_mode | bool %}
+            - name: awx-devel
+              mountPath: "/awx_devel"
+{% endif %}
+          env:
+            - name: SUPERVISOR_WEB_CONFIG_PATH
+              value: "/etc/supervisor_rsyslog.conf"
+{% if development_mode | bool %}
+            - name: AWX_KUBE_DEVEL
+              value: "1"
+{% endif %}
+{% if web_tolerations %}
+      tolerations:
+        {{ web_tolerations| indent(width=8) }}
+{% elif tolerations %}
+      tolerations:
+        {{ tolerations| indent(width=8) }}
+{% endif %}
+{% if web_affinity %}
+      affinity:
+        {{ web_affinity | to_nice_yaml | indent(width=8) }}
+{% elif affinity %}
+      affinity:
+        {{ affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
      volumes:
 {% if bundle_ca_crt %}
        - name: "ca-trust-extracted"
@@ -257,14 +304,10 @@ spec:
          emptyDir: {}
        - name: {{ ansible_operator_meta.name }}-redis-data
          emptyDir: {}
-        - name: supervisor-socket
-          emptyDir: {}
        - name: rsyslog-socket
          emptyDir: {}
        - name: receptor-socket
          emptyDir: {}
-        - name: rsyslog-dir
-          emptyDir: {}
        - name: {{ ansible_operator_meta.name }}-receptor-config
          configMap:
            name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'