From d64c34f8a48742d75dda810e0be7d71ed9d787a6 Mon Sep 17 00:00:00 2001 From: Hao Liu <44379968+TheRealHaoLiu@users.noreply.github.com> Date: Fri, 9 Sep 2022 15:13:05 -0400 Subject: [PATCH] Add receptor firewall rules to control nodes (#1012) Support external execution nodes - Allow receptor.conf to be editable at runtime - Create CA cert and key as a k8s secret - Create work signing RSA keypair as a k8s secret - Setup volume mounts for containers to have access to the needed Receptor keys / certs to facilitate generating the install bundle for a new execution node - added firewall rule, work signing and tls cert configuration to default receptor.conf The volume mount changes in this PR fulfill the following: - `receptor.conf` need to be shared between task container and ee container - **task** container writes the `receptor.conf` - **ee** consume the `receptor.conf` - receptor ca cert/key need to be mounted by both ee container and web container - **ee** container need the ca cert - **web** container will need the ca key to sign client cert for remote execution node - **web** container will need the ca cert to generate install bundle for remote execution node - receptor work private/public key need to be mounted by both ee container and web container - **ee** container need to private key to sign the work - **web** container need the public key to generate install bundle for remote execution node - **task** container need the private key to sign the work Signed-off-by: Hao Liu Co-Authored-By: Seth Foster Co-Authored-By: Shane McDonald Signed-off-by: Hao Liu Co-authored-by: Shane McDonald Co-authored-by: Seth Foster --- Dockerfile | 6 ++ roles/installer/defaults/main.yml | 5 +- .../tasks/resources_configuration.yml | 89 +++++++++++++++++++ .../templates/configmaps/config.yaml.j2 | 22 +++-- .../templates/deployments/deployment.yaml.j2 | 63 +++++++++++-- .../secrets/receptor_ca_secret.yaml.j2 | 15 ++++ .../receptor_work_signing_secret.yaml.j2 | 15 ++++ 7 files changed, 199 insertions(+), 16 deletions(-) create mode 100644 roles/installer/templates/secrets/receptor_ca_secret.yaml.j2 create mode 100644 roles/installer/templates/secrets/receptor_work_signing_secret.yaml.j2 diff --git a/Dockerfile b/Dockerfile index d0a9a43f..a272ef9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,11 @@ FROM quay.io/operator-framework/ansible-operator:v1.22.2 +USER 0 + +RUN dnf install -y openssl + +USER 1001 + ARG DEFAULT_AWX_VERSION ARG OPERATOR_VERSION ENV DEFAULT_AWX_VERSION=${DEFAULT_AWX_VERSION} diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index e9996f3d..6fc9ebef 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -131,8 +131,6 @@ _redis_image: docker.io/redis _redis_image_version: 7 _postgres_image: postgres _postgres_image_version: 13 -_init_container_image: quay.io/centos/centos -_init_container_image_version: stream8 image_pull_policy: IfNotPresent image_pull_secrets: [] @@ -158,6 +156,9 @@ ee_images: _control_plane_ee_image: quay.io/ansible/awx-ee:latest +_init_container_image: "{{ _control_plane_ee_image.split(':')[0] }}" +_init_container_image_version: "{{ _control_plane_ee_image.split(':')[1] }}" + create_preload_data: true replicas: "1" diff --git a/roles/installer/tasks/resources_configuration.yml b/roles/installer/tasks/resources_configuration.yml index a24b8d6e..7ce87217 100644 --- a/roles/installer/tasks/resources_configuration.yml +++ b/roles/installer/tasks/resources_configuration.yml @@ -27,6 +27,95 @@ set_fact: _control_plane_ee_image: "{{ _custom_control_plane_ee_image | default(lookup('env', 'RELATED_IMAGE_CONTROL_PLANE_EE')) | default(_control_plane_ee_image, true) }}" +- name: Check for Receptor CA Secret + k8s_info: + kind: Secret + namespace: '{{ ansible_operator_meta.namespace }}' + name: '{{ ansible_operator_meta.name }}-receptor-ca' + register: _receptor_ca + no_log: "{{ no_log }}" + +- name: Create Receptor Mesh CA + block: + - name: Create tempfile for receptor-ca.key + tempfile: + state: file + suffix: .key + register: _receptor_ca_key_file + - name: Generate Receptor CA key + command: | + openssl genrsa -out {{ _receptor_ca_key_file.path }} 4096 + no_log: "{{ no_log }}" + - name: Create tempfile for receptor-ca.crt + tempfile: + state: file + suffix: .crt + register: _receptor_ca_crt_file + - name: Generate Receptor CA cert + command: | + openssl req -x509 -new -nodes -key {{ _receptor_ca_key_file.path }} \ + -subj "/CN={{ ansible_operator_meta.name }} Receptor Root CA" \ + -sha256 -days 3650 -out {{ _receptor_ca_crt_file.path }} + no_log: "{{ no_log }}" + - name: Create Receptor CA secret + k8s: + apply: true + definition: "{{ lookup('template', 'secrets/receptor_ca_secret.yaml.j2') }}" + no_log: "{{ no_log }}" + - name: Remove tempfiles + file: + path: "{{ item }}" + state: absent + loop: + - "{{ _receptor_ca_key_file.path }}" + - "{{ _receptor_ca_crt_file.path }}" + when: not _receptor_ca['resources'] | default([]) | length + +- name: Check for Receptor work signing Secret + k8s_info: + kind: Secret + namespace: '{{ ansible_operator_meta.namespace }}' + name: '{{ ansible_operator_meta.name }}-receptor-work-signing' + register: _receptor_work_signing + no_log: "{{ no_log }}" + +- name: Generate Receptor work signing RSA key pair + block: + - name: Create tempfile for receptor work signing private key + tempfile: + state: file + suffix: .pem + register: _receptor_work_signing_private_key_file + - name: Generate Receptor work signing private key + command: | + openssl genrsa -out {{ _receptor_work_signing_private_key_file.path }} 4096 + no_log: "{{ no_log }}" + - name: Create tempfile for receptor work signing public key + tempfile: + state: file + suffix: .pem + register: _receptor_work_signing_public_key_file + - name: Generate Receptor work signing public key + command: | + openssl rsa \ + -in {{ _receptor_work_signing_private_key_file.path }} \ + -out {{ _receptor_work_signing_public_key_file.path }} \ + -outform PEM -pubout + no_log: "{{ no_log }}" + - name: Create Receptor work signing Secret + k8s: + apply: true + definition: "{{ lookup('template', 'secrets/receptor_work_signing_secret.yaml.j2') }}" + no_log: "{{ no_log }}" + - name: Remove tempfiles + file: + path: "{{ item }}" + state: absent + loop: + - "{{ _receptor_work_signing_private_key_file.path }}" + - "{{ _receptor_work_signing_public_key_file.path }}" + when: not _receptor_work_signing['resources'] | default([]) | length + - name: Apply Resources k8s: apply: yes diff --git a/roles/installer/templates/configmaps/config.yaml.j2 b/roles/installer/templates/configmaps/config.yaml.j2 index 12a29123..05886a52 100644 --- a/roles/installer/templates/configmaps/config.yaml.j2 +++ b/roles/installer/templates/configmaps/config.yaml.j2 @@ -236,30 +236,38 @@ data: receptor_conf: | --- - log-level: debug - + - local-only: null + - node: + firewallrules: + - action: reject + tonode: HOSTNAME + toservice: control - control-service: service: control filename: /var/run/receptor/receptor.sock - permissions: 0660 - - - local-only: - + permissions: '0660' - work-command: worktype: local command: ansible-runner params: worker allowruntimeparams: true - - work-kubernetes: worktype: kubernetes-runtime-auth authmethod: runtime allowruntimeauth: true allowruntimepod: true allowruntimeparams: true - - work-kubernetes: worktype: kubernetes-incluster-auth authmethod: incluster allowruntimeauth: true allowruntimepod: true allowruntimeparams: true + - tls-client: + cert: /etc/receptor/tls/receptor.crt + key: /etc/receptor/tls/receptor.key + name: tlsclient + rootcas: /etc/receptor/tls/ca/receptor-ca.crt + - work-signing: + privatekey: /etc/receptor/signing/work-private-key.pem + tokenexpiration: 1m diff --git a/roles/installer/templates/deployments/deployment.yaml.j2 b/roles/installer/templates/deployments/deployment.yaml.j2 index e5ad9290..d2b02c11 100644 --- a/roles/installer/templates/deployments/deployment.yaml.j2 +++ b/roles/installer/templates/deployments/deployment.yaml.j2 @@ -39,7 +39,6 @@ spec: priorityClassName: '{{ control_plane_priority_class }}' {% endif %} initContainers: -{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %} - name: init image: '{{ _init_container_image }}' imagePullPolicy: '{{ image_pull_policy }}' @@ -47,6 +46,9 @@ spec: - /bin/sh - -c - | + hostname=$MY_POD_NAME + receptor --cert-makereq bits=2048 commonname=$hostname dnsname=$hostname nodeid=$hostname outreq=/etc/receptor/tls/receptor.req outkey=/etc/receptor/tls/receptor.key + receptor --cert-signreq req=/etc/receptor/tls/receptor.req cacert=/etc/receptor/tls/ca/receptor-ca.crt cakey=/etc/receptor/tls/ca/receptor-ca.key outcert=/etc/receptor/tls/receptor.crt verify=yes {% if bundle_ca_crt %} mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2} update-ca-trust @@ -58,7 +60,17 @@ spec: {% if init_container_extra_commands %} {{ init_container_extra_commands | indent(width=14) }} {% endif %} + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name volumeMounts: + - name: "{{ ansible_operator_meta.name }}-receptor-ca" + mountPath: "/etc/receptor/tls/ca" + readOnly: true + - name: "{{ ansible_operator_meta.name }}-receptor-tls" + mountPath: "/etc/receptor/tls/" {% if bundle_ca_crt %} - name: "ca-trust-extracted" mountPath: "/etc/pki/ca-trust/extracted" @@ -73,7 +85,6 @@ spec: {% endif %} {% if init_container_extra_volume_mounts -%} {{ init_container_extra_volume_mounts | indent(width=12, first=True) }} -{% endif %} {% endif %} containers: - image: '{{ _redis_image }}' @@ -163,6 +174,13 @@ spec: mountPath: "/var/lib/awx/rsyslog" - name: "{{ ansible_operator_meta.name }}-projects" mountPath: "/var/lib/awx/projects" + - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" + mountPath: "/etc/receptor/signing/work-public-key.pem" + subPath: "work-public-key.pem" + readOnly: true + - name: "{{ ansible_operator_meta.name }}-receptor-ca" + mountPath: "/etc/receptor/tls/ca" + readOnly: true {% if development_mode | bool %} - name: awx-devel mountPath: "/awx_devel" @@ -236,8 +254,10 @@ spec: - name: rsyslog-dir mountPath: "/var/lib/awx/rsyslog" - name: "{{ ansible_operator_meta.name }}-receptor-config" - mountPath: "/etc/receptor/receptor.conf" - subPath: receptor.conf + mountPath: "/etc/receptor/" + - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" + mountPath: "/etc/receptor/signing/work-private-key.pem" + subPath: "work-private-key.pem" readOnly: true - name: receptor-socket mountPath: "/var/run/receptor" @@ -279,7 +299,15 @@ spec: name: '{{ ansible_operator_meta.name }}-ee' imagePullPolicy: '{{ image_pull_policy }}' resources: {{ ee_resource_requirements }} - args: ['receptor', '--config', '/etc/receptor/receptor.conf'] + args: + - /bin/sh + - -c + - | + if [ ! -f /etc/receptor/receptor.conf ]; then + cp /etc/receptor/receptor-default.conf /etc/receptor/receptor.conf + sed -i "s/HOSTNAME/$HOSTNAME/g" /etc/receptor/receptor.conf + fi + exec receptor --config /etc/receptor/receptor.conf volumeMounts: {% if bundle_ca_crt %} - name: "ca-trust-extracted" @@ -289,10 +317,21 @@ spec: subPath: bundle-ca.crt readOnly: true {% endif %} - - name: "{{ ansible_operator_meta.name }}-receptor-config" - mountPath: "/etc/receptor/receptor.conf" + - name: "{{ ansible_operator_meta.name }}-default-receptor-config" + mountPath: "/etc/receptor/receptor-default.conf" subPath: receptor.conf + - name: "{{ ansible_operator_meta.name }}-receptor-config" + mountPath: "/etc/receptor/" + - name: "{{ ansible_operator_meta.name }}-receptor-ca" + mountPath: "/etc/receptor/tls/ca/receptor-ca.crt" + subPath: "receptor-ca.crt" readOnly: true + - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" + mountPath: "/etc/receptor/signing/work-private-key.pem" + subPath: "work-private-key.pem" + readOnly: true + - name: "{{ ansible_operator_meta.name }}-receptor-tls" + mountPath: "/etc/receptor/tls/" - name: receptor-socket mountPath: "/var/run/receptor" - name: "{{ ansible_operator_meta.name }}-projects" @@ -370,6 +409,14 @@ spec: path: 'ldap.py' - key: execution_environments.py path: 'execution_environments.py' + - name: "{{ ansible_operator_meta.name }}-receptor-tls" + emptyDir: {} + - name: "{{ ansible_operator_meta.name }}-receptor-ca" + secret: + secretName: "{{ ansible_operator_meta.name }}-receptor-ca" + - name: "{{ ansible_operator_meta.name }}-receptor-work-signing" + secret: + secretName: "{{ ansible_operator_meta.name }}-receptor-work-signing" - name: "{{ secret_key_secret_name }}" secret: secretName: '{{ secret_key_secret_name }}' @@ -407,6 +454,8 @@ spec: - name: rsyslog-dir emptyDir: {} - name: {{ ansible_operator_meta.name }}-receptor-config + emptyDir: {} + - name: {{ ansible_operator_meta.name }}-default-receptor-config configMap: name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap' items: diff --git a/roles/installer/templates/secrets/receptor_ca_secret.yaml.j2 b/roles/installer/templates/secrets/receptor_ca_secret.yaml.j2 new file mode 100644 index 00000000..6a6d0c05 --- /dev/null +++ b/roles/installer/templates/secrets/receptor_ca_secret.yaml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: '{{ ansible_operator_meta.name }}-receptor-ca' + namespace: '{{ ansible_operator_meta.namespace }}' + labels: + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}' + app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}' + app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' + app.kubernetes.io/component: '{{ deployment_type }}' + app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}' +data: + receptor-ca.crt: '{{ lookup('file', '{{ _receptor_ca_crt_file.path }}') | b64encode }}' + receptor-ca.key: '{{ lookup('file', '{{ _receptor_ca_key_file.path }}') | b64encode }}' diff --git a/roles/installer/templates/secrets/receptor_work_signing_secret.yaml.j2 b/roles/installer/templates/secrets/receptor_work_signing_secret.yaml.j2 new file mode 100644 index 00000000..1b0acac4 --- /dev/null +++ b/roles/installer/templates/secrets/receptor_work_signing_secret.yaml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: '{{ ansible_operator_meta.name }}-receptor-work-signing' + namespace: '{{ ansible_operator_meta.namespace }}' + labels: + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}' + app.kubernetes.io/part-of: '{{ ansible_operator_meta.name }}' + app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' + app.kubernetes.io/component: '{{ deployment_type }}' + app.kubernetes.io/operator-version: '{{ lookup("env", "OPERATOR_VERSION") }}' +data: + work-private-key.pem: '{{ lookup('file', '{{ _receptor_work_signing_private_key_file.path }}') | b64encode }}' + work-public-key.pem: '{{ lookup('file', '{{ _receptor_work_signing_public_key_file.path }}') | b64encode }}'