Add receptor firewall rules to control nodes (#1012)

Support external execution nodes

- Allow receptor.conf to be editable at runtime
- Create CA cert and key as a k8s secret
- Create work signing RSA keypair as a k8s secret
- Setup volume mounts for containers to have access to the needed
  Receptor keys / certs to facilitate generating the install bundle
  for a new execution node
- added firewall rule, work signing and tls cert configuration to default receptor.conf

The volume mount changes in this PR fulfill the following:
- `receptor.conf` need to be shared between task container and ee container
  - **task** container writes the `receptor.conf`
  - **ee** consume the `receptor.conf`
- receptor ca cert/key need to be mounted by both ee container and web container
  - **ee** container need the ca cert
  - **web** container will need the ca key to sign client cert for remote execution node
  - **web** container will need the ca cert to generate install bundle for remote execution node
- receptor work private/public key need to be mounted by both ee container and web container
  - **ee** container need to private key to sign the work
  - **web** container need the public key to generate install bundle  for remote execution node
  - **task** container need the private key to sign the work

Signed-off-by: Hao Liu <haoli@redhat.com>
Co-Authored-By: Seth Foster <fosterbseth@gmail.com>
Co-Authored-By: Shane McDonald <me@shanemcd.com>

Signed-off-by: Hao Liu <haoli@redhat.com>
Co-authored-by: Shane McDonald <me@shanemcd.com>
Co-authored-by: Seth Foster <fosterbseth@gmail.com>

Dockerfile

@@ -1,5 +1,11 @@
 FROM quay.io/operator-framework/ansible-operator:v1.22.2
 
+USER 0
+
+RUN dnf install -y openssl
+
+USER 1001
+
 ARG DEFAULT_AWX_VERSION
 ARG OPERATOR_VERSION
 ENV DEFAULT_AWX_VERSION=${DEFAULT_AWX_VERSION}

roles/installer/defaults/main.yml

@@ -131,8 +131,6 @@ _redis_image: docker.io/redis
 _redis_image_version: 7
 _postgres_image: postgres
 _postgres_image_version: 13
-_init_container_image: quay.io/centos/centos
-_init_container_image_version: stream8
 image_pull_policy: IfNotPresent
 image_pull_secrets: []
 
@@ -158,6 +156,9 @@ ee_images:
 
 _control_plane_ee_image: quay.io/ansible/awx-ee:latest
 
+_init_container_image: "{{ _control_plane_ee_image.split(':')[0] }}"
+_init_container_image_version: "{{ _control_plane_ee_image.split(':')[1] }}"
+
 create_preload_data: true
 
 replicas: "1"

roles/installer/tasks/resources_configuration.yml

@@ -27,6 +27,95 @@
   set_fact:
     _control_plane_ee_image: "{{ _custom_control_plane_ee_image | default(lookup('env', 'RELATED_IMAGE_CONTROL_PLANE_EE')) | default(_control_plane_ee_image, true) }}"
 
+- name: Check for Receptor CA Secret
+  k8s_info:
+    kind: Secret
+    namespace: '{{ ansible_operator_meta.namespace }}'
+    name: '{{ ansible_operator_meta.name }}-receptor-ca'
+  register: _receptor_ca
+  no_log: "{{ no_log }}"
+
+- name: Create Receptor Mesh CA
+  block:
+    - name: Create tempfile for receptor-ca.key
+      tempfile:
+        state: file
+        suffix: .key
+      register: _receptor_ca_key_file
+    - name: Generate Receptor CA key
+      command: |
+        openssl genrsa -out {{ _receptor_ca_key_file.path }} 4096
+      no_log: "{{ no_log }}"
+    - name: Create tempfile for receptor-ca.crt
+      tempfile:
+        state: file
+        suffix: .crt
+      register: _receptor_ca_crt_file
+    - name: Generate Receptor CA cert
+      command: |
+        openssl req -x509 -new -nodes -key {{ _receptor_ca_key_file.path }} \
+          -subj "/CN={{ ansible_operator_meta.name }} Receptor Root CA" \
+          -sha256 -days 3650 -out {{ _receptor_ca_crt_file.path }}
+      no_log: "{{ no_log }}"
+    - name: Create Receptor CA secret
+      k8s:
+        apply: true
+        definition: "{{ lookup('template', 'secrets/receptor_ca_secret.yaml.j2') }}"
+      no_log: "{{ no_log }}"
+    - name: Remove tempfiles
+      file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "{{ _receptor_ca_key_file.path }}"
+        - "{{ _receptor_ca_crt_file.path }}"
+  when: not _receptor_ca['resources'] | default([]) | length
+
+- name: Check for Receptor work signing Secret
+  k8s_info:
+    kind: Secret
+    namespace: '{{ ansible_operator_meta.namespace }}'
+    name: '{{ ansible_operator_meta.name }}-receptor-work-signing'
+  register: _receptor_work_signing
+  no_log: "{{ no_log }}"
+
+- name: Generate Receptor work signing RSA key pair
+  block:
+    - name: Create tempfile for receptor work signing private key
+      tempfile:
+        state: file
+        suffix: .pem
+      register: _receptor_work_signing_private_key_file
+    - name: Generate Receptor work signing private key
+      command: |
+        openssl genrsa -out {{ _receptor_work_signing_private_key_file.path }} 4096
+      no_log: "{{ no_log }}"
+    - name: Create tempfile for receptor work signing public key
+      tempfile:
+        state: file
+        suffix: .pem
+      register: _receptor_work_signing_public_key_file
+    - name: Generate Receptor work signing public key
+      command: |
+        openssl rsa \
+          -in {{ _receptor_work_signing_private_key_file.path }} \
+          -out {{ _receptor_work_signing_public_key_file.path }} \
+          -outform PEM -pubout
+      no_log: "{{ no_log }}"
+    - name: Create Receptor work signing Secret
+      k8s:
+        apply: true
+        definition: "{{ lookup('template', 'secrets/receptor_work_signing_secret.yaml.j2') }}"
+      no_log: "{{ no_log }}"
+    - name: Remove tempfiles
+      file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "{{ _receptor_work_signing_private_key_file.path }}"
+        - "{{ _receptor_work_signing_public_key_file.path }}"
+  when: not _receptor_work_signing['resources'] | default([]) | length
+
 - name: Apply Resources
   k8s:
     apply: yes

roles/installer/templates/configmaps/config.yaml.j2

@@ -236,30 +236,38 @@ data:
   receptor_conf: |
     ---
     - log-level: debug
-
+    - local-only: null
+    - node:
+       firewallrules:
+        - action: reject
+          tonode: HOSTNAME
+          toservice: control
     - control-service:
         service: control
         filename: /var/run/receptor/receptor.sock
-        permissions: 0660
+        permissions: '0660'
-
-    - local-only:
-
     - work-command:
         worktype: local
         command: ansible-runner
         params: worker
         allowruntimeparams: true
-
     - work-kubernetes:
         worktype: kubernetes-runtime-auth
         authmethod: runtime
         allowruntimeauth: true
         allowruntimepod: true
         allowruntimeparams: true
-
     - work-kubernetes:
         worktype: kubernetes-incluster-auth
         authmethod: incluster
         allowruntimeauth: true
         allowruntimepod: true
         allowruntimeparams: true
+    - tls-client:
+        cert: /etc/receptor/tls/receptor.crt
+        key: /etc/receptor/tls/receptor.key
+        name: tlsclient
+        rootcas: /etc/receptor/tls/ca/receptor-ca.crt
+    - work-signing:
+        privatekey: /etc/receptor/signing/work-private-key.pem
+        tokenexpiration: 1m

roles/installer/templates/deployments/deployment.yaml.j2

@@ -39,7 +39,6 @@ spec:
       priorityClassName: '{{ control_plane_priority_class }}'
 {% endif %}
       initContainers:
-{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %}
         - name: init
           image: '{{ _init_container_image }}'
           imagePullPolicy: '{{ image_pull_policy }}'
@@ -47,6 +46,9 @@ spec:
             - /bin/sh
             - -c
             - |
+              hostname=$MY_POD_NAME
+              receptor --cert-makereq bits=2048 commonname=$hostname dnsname=$hostname nodeid=$hostname outreq=/etc/receptor/tls/receptor.req outkey=/etc/receptor/tls/receptor.key
+              receptor --cert-signreq req=/etc/receptor/tls/receptor.req cacert=/etc/receptor/tls/ca/receptor-ca.crt cakey=/etc/receptor/tls/ca/receptor-ca.key outcert=/etc/receptor/tls/receptor.crt verify=yes
 {% if bundle_ca_crt  %}
               mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2}
               update-ca-trust
@@ -58,7 +60,17 @@ spec:
 {% if init_container_extra_commands %}
               {{ init_container_extra_commands | indent(width=14) }}
 {% endif %}
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           volumeMounts:
+            - name: "{{ ansible_operator_meta.name }}-receptor-ca"
+              mountPath: "/etc/receptor/tls/ca"
+              readOnly: true
+            - name: "{{ ansible_operator_meta.name }}-receptor-tls"
+              mountPath: "/etc/receptor/tls/"
 {% if bundle_ca_crt  %}
             - name: "ca-trust-extracted"
               mountPath: "/etc/pki/ca-trust/extracted"
@@ -73,7 +85,6 @@ spec:
 {% endif %}
 {% if init_container_extra_volume_mounts -%}
             {{ init_container_extra_volume_mounts | indent(width=12, first=True) }}
-{% endif %}
 {% endif %}
       containers:
         - image: '{{ _redis_image }}'
@@ -163,6 +174,13 @@ spec:
               mountPath: "/var/lib/awx/rsyslog"
             - name: "{{ ansible_operator_meta.name }}-projects"
               mountPath: "/var/lib/awx/projects"
+            - name: "{{ ansible_operator_meta.name }}-receptor-work-signing"
+              mountPath: "/etc/receptor/signing/work-public-key.pem"
+              subPath: "work-public-key.pem"
+              readOnly: true
+            - name: "{{ ansible_operator_meta.name }}-receptor-ca"
+              mountPath: "/etc/receptor/tls/ca"
+              readOnly: true
 {% if development_mode | bool %}
             - name: awx-devel
               mountPath: "/awx_devel"
@@ -236,8 +254,10 @@ spec:
             - name: rsyslog-dir
               mountPath: "/var/lib/awx/rsyslog"
             - name: "{{ ansible_operator_meta.name }}-receptor-config"
-              mountPath: "/etc/receptor/receptor.conf"
+              mountPath: "/etc/receptor/"
-              subPath: receptor.conf
+            - name: "{{ ansible_operator_meta.name }}-receptor-work-signing"
+              mountPath: "/etc/receptor/signing/work-private-key.pem"
+              subPath: "work-private-key.pem"
               readOnly: true
             - name: receptor-socket
               mountPath: "/var/run/receptor"
@@ -279,7 +299,15 @@ spec:
           name: '{{ ansible_operator_meta.name }}-ee'
           imagePullPolicy: '{{ image_pull_policy }}'
           resources: {{ ee_resource_requirements }}
-          args: ['receptor', '--config', '/etc/receptor/receptor.conf']
+          args:
+            - /bin/sh
+            - -c
+            - |
+              if [ ! -f /etc/receptor/receptor.conf ]; then
+                cp /etc/receptor/receptor-default.conf /etc/receptor/receptor.conf
+                sed -i "s/HOSTNAME/$HOSTNAME/g" /etc/receptor/receptor.conf
+              fi
+              exec receptor --config /etc/receptor/receptor.conf
           volumeMounts:
 {% if bundle_ca_crt %}
             - name: "ca-trust-extracted"
@@ -289,10 +317,21 @@ spec:
               subPath: bundle-ca.crt
               readOnly: true
 {% endif %}
-            - name: "{{ ansible_operator_meta.name }}-receptor-config"
+            - name: "{{ ansible_operator_meta.name }}-default-receptor-config"
-              mountPath: "/etc/receptor/receptor.conf"
+              mountPath: "/etc/receptor/receptor-default.conf"
               subPath: receptor.conf
+            - name: "{{ ansible_operator_meta.name }}-receptor-config"
+              mountPath: "/etc/receptor/"
+            - name: "{{ ansible_operator_meta.name }}-receptor-ca"
+              mountPath: "/etc/receptor/tls/ca/receptor-ca.crt"
+              subPath: "receptor-ca.crt"
               readOnly: true
+            - name: "{{ ansible_operator_meta.name }}-receptor-work-signing"
+              mountPath: "/etc/receptor/signing/work-private-key.pem"
+              subPath: "work-private-key.pem"
+              readOnly: true
+            - name: "{{ ansible_operator_meta.name }}-receptor-tls"
+              mountPath: "/etc/receptor/tls/"
             - name: receptor-socket
               mountPath: "/var/run/receptor"
             - name: "{{ ansible_operator_meta.name }}-projects"
@@ -370,6 +409,14 @@ spec:
                 path: 'ldap.py'
               - key: execution_environments.py
                 path: 'execution_environments.py'
+        - name: "{{ ansible_operator_meta.name }}-receptor-tls"
+          emptyDir: {}
+        - name: "{{ ansible_operator_meta.name }}-receptor-ca"
+          secret:
+            secretName: "{{ ansible_operator_meta.name }}-receptor-ca"
+        - name: "{{ ansible_operator_meta.name }}-receptor-work-signing"
+          secret:
+            secretName: "{{ ansible_operator_meta.name }}-receptor-work-signing"
         - name: "{{ secret_key_secret_name }}"
           secret:
             secretName: '{{ secret_key_secret_name }}'
@@ -407,6 +454,8 @@ spec:
         - name: rsyslog-dir
           emptyDir: {}
         - name: {{ ansible_operator_meta.name }}-receptor-config
+          emptyDir: {}
+        - name: {{ ansible_operator_meta.name }}-default-receptor-config
           configMap:
             name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap'
             items:

roles/installer/templates/secrets/receptor_ca_secret.yaml.j2

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ ansible_operator_meta.name }}-receptor-ca'
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
+    app.kubernetes.io/component: '{{ deployment_type }}'
+    app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}'
+data:
+  receptor-ca.crt: '{{ lookup('file', '{{ _receptor_ca_crt_file.path }}') | b64encode }}'
+  receptor-ca.key: '{{ lookup('file', '{{ _receptor_ca_key_file.path }}') | b64encode }}'

roles/installer/templates/secrets/receptor_work_signing_secret.yaml.j2

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ ansible_operator_meta.name }}-receptor-work-signing'
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    app.kubernetes.io/name: '{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}'
+    app.kubernetes.io/managed-by: '{{ deployment_type }}-operator'
+    app.kubernetes.io/component: '{{ deployment_type }}'
+    app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}'
+data:
+  work-private-key.pem: '{{ lookup('file', '{{ _receptor_work_signing_private_key_file.path }}') | b64encode }}'
+  work-public-key.pem: '{{ lookup('file', '{{ _receptor_work_signing_public_key_file.path }}') | b64encode }}'