From 24916c6fa10973c0b221889ca82c319ad7544251 Mon Sep 17 00:00:00 2001 From: Marcelo Moreira de Mello Date: Thu, 17 Jun 2021 23:32:52 -0400 Subject: [PATCH] Using fsGroup and extends propers via InitContainer (NFS) --- README.md | 135 ++++++++++-------- ansible/templates/crd.yml.j2 | 6 + deploy/awx-operator.yaml | 6 + deploy/crds/awx_v1beta1_crd.yaml | 6 + .../awx-operator.clusterserviceversion.yaml | 12 ++ .../manifests/awx.ansible.com_awxs_crd.yaml | 6 + roles/installer/defaults/main.yml | 16 +++ roles/installer/templates/deployment.yaml.j2 | 24 +++- 8 files changed, 151 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index aae3ed84..fafabf7d 100644 --- a/README.md +++ b/README.md @@ -572,60 +572,20 @@ spec: In a scenario where custom volumes and volume mounts are required to either overwrite defaults or mount configuration files. -| Name | Description | Default | -| ------------------------------ | -------------------------------------------------------- | ------- | -| extra_volumes | Specify extra volumes to add to the application pod | '' | -| web_extra_volume_mounts | Specify volume mounts to be added to Web container | '' | -| task_extra_volume_mounts | Specify volume mounts to be added to Task container | '' | -| ee_extra_volume_mounts | Specify volume mounts to be added to Execution container | '' | +| Name | Description | Default | +| --------------------------------- | -------------------------------------------------------- | ------- | +| extra_volumes | Specify extra volumes to add to the application pod | '' | +| web_extra_volume_mounts | Specify volume mounts to be added to Web container | '' | +| task_extra_volume_mounts | Specify volume mounts to be added to Task container | '' | +| ee_extra_volume_mounts | Specify volume mounts to be added to Execution container | '' | +| init_container_extra_volume_mounts| Specify volume mounts to be added to Init container | '' | +| init_container_extra_commands | Specify additional commands for Init container | '' | + > :warning: The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec). Example configuration for ConfigMap -#### Default execution environments from private registries - -In order to register default execution environments from private registries, the Custom Resource needs to know about the pull credentials. Those credentials should be stored as a secret and specified as `ee_pull_credentials_secret` at the CR spec level. Instance initialization will register a `Container registry` type credential on the deployed instance and assign it to the registered default execution environments. - -The secret should be formated as follows: - -```yaml ---- -apiVersion: v1 -kind: Secret -metadata: - name: my-ee-pull-credentials - namespace: -stringData: - url: - username: - password: - ssl_verify: -type: Opaque -``` - -##### Control plane EE from private registry -The images listed in "ee_images" will be added as globally available Execution Environments. The "control_plane_ee_image" will be used to run project updates. In order to use a private image for any of these you'll need to use `image_pull_secret` to provide a k8s pull secret to access it. Currently the same secret is used for any of these images supplied at install time. - -You can create `image_pull_secret` -``` -kubectl create secret -cp-pull-credentials regcred --docker-server= --docker-username= --docker-password= --docker-email= -``` -If you need more control (for example, to set a namespace or a label on the new secret) then you can customise the Secret before storing it - -```yaml ---- -apiVersion: v1 -kind: Secret -metadata: - name: -cp-pull-credentials - namespace: -data: - .dockerconfigjson: -type: kubernetes.io/dockerconfigjson -``` -Example spec file extra-config - ```yaml --- apiVersion: v1 @@ -649,16 +609,6 @@ Example spec file for volumes and volume mounts --- spec: ... - ee_extra_volume_mounts: | - - name: ansible-cfg - mountPath: /etc/ansible/ansible.cfg - subPath: ansible.cfg - - task_extra_volume_mounts: | - - name: custom-py - mountPath: /etc/tower/conf.d/custom.py - subPath: custom.py - extra_volumes: | - name: ansible-cfg configMap: @@ -674,11 +624,78 @@ Example spec file for volumes and volume mounts - key: custom.py path: custom.py name: -extra-config + - name: shared-volume + persistentVolumeClaim: + claimName: my-external-volume-claim + init_container_extra_volume_mounts: | + - name: shared-volume + mountPath: /shared + + init_container_extra_commands: | + # set proper permissions (rwx) for the awx user + chmod 775 /shared + chgrp 1000 /shared + + ee_extra_volume_mounts: | + - name: ansible-cfg + mountPath: /etc/ansible/ansible.cfg + subPath: ansible.cfg + + task_extra_volume_mounts: | + - name: custom-py + mountPath: /etc/tower/conf.d/custom.py + subPath: custom.py + - name: shared-volume + mountPath: /shared ``` > :warning: **Volume and VolumeMount names cannot contain underscores(_)** +#### Default execution environments from private registries + +In order to register default execution environments from private registries, the Custom Resource needs to know about the pull credentials. Those credentials should be stored as a secret and either specified as `ee_pull_credentials_secret` at the CR spec level, or simply be present on the namespace under the name `-ee-pull-credentials` . Instance initialization will register a `Container registry` type credential on the deployed instance and assign it to the registered default execution environments. + +The secret should be formated as follows: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: -ee-pull-credentials + namespace: +stringData: + url: + username: + password: + ssl_verify: +type: Opaque +``` + +##### Control plane ee from private registry +The images listed in "ee_images" will be added as globally available Execution Environments. The "control_plane_ee_image" will be used to run project updates. In order to use a private image for any of these you'll need to use `image_pull_secret` to provide a k8s pull secret to access it. Currently the same secret is used for any of these images supplied at install time. + +You can create `image_pull_secret` +``` +kubectl create secret -cp-pull-credentials regcred --docker-server= --docker-username= --docker-password= --docker-email= +``` +If you need more control (for example, to set a namespace or a label on the new secret) then you can customise the Secret before storing it + +Example spec file extra-config + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: -cp-pull-credentials + namespace: +data: + .dockerconfigjson: +type: kubernetes.io/dockerconfigjson +``` + #### Exporting Environment Variables to Containers If you need to export custom environment variables to your containers. diff --git a/ansible/templates/crd.yml.j2 b/ansible/templates/crd.yml.j2 index 961ade74..a8c489be 100644 --- a/ansible/templates/crd.yml.j2 +++ b/ansible/templates/crd.yml.j2 @@ -291,6 +291,12 @@ spec: init_container_image_version: description: Init container image version to use type: string + init_container_extra_commands: + description: Extra commands for the init container + type: string + init_container_extra_volume_mounts: + description: Specify volume mounts to be added to the init container + type: string postgres_image: description: Registry path to the PostgreSQL container to use type: string diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index 89cb5226..92f102ba 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -293,6 +293,12 @@ spec: init_container_image_version: description: Init container image version to use type: string + init_container_extra_commands: + description: Extra commands for the init container + type: string + init_container_extra_volume_mounts: + description: Specify volume mounts to be added to the init container + type: string postgres_image: description: Registry path to the PostgreSQL container to use type: string diff --git a/deploy/crds/awx_v1beta1_crd.yaml b/deploy/crds/awx_v1beta1_crd.yaml index 961ade74..a8c489be 100644 --- a/deploy/crds/awx_v1beta1_crd.yaml +++ b/deploy/crds/awx_v1beta1_crd.yaml @@ -291,6 +291,12 @@ spec: init_container_image_version: description: Init container image version to use type: string + init_container_extra_commands: + description: Extra commands for the init container + type: string + init_container_extra_volume_mounts: + description: Specify volume mounts to be added to the init container + type: string postgres_image: description: Registry path to the PostgreSQL container to use type: string diff --git a/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml b/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml index d43c8d1c..ab6bc99b 100644 --- a/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml @@ -565,6 +565,18 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:hidden + - description: Specify Extra commands for the Init container + displayName: Init Container Extra Commands + path: init_container_extra_commands + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden + - description: Specify volume mounts to be added to Init container + displayName: Init Container Extra Volume Mounts + path: init_container_extra_volume_mounts + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden - description: Secret where can be found the trusted Certificate Authority Bundle path: bundle_cacert_secret x-descriptors: diff --git a/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml b/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml index 6c174639..21476433 100644 --- a/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml +++ b/deploy/olm-catalog/awx-operator/manifests/awx.ansible.com_awxs_crd.yaml @@ -158,6 +158,12 @@ spec: init_container_image_version: description: Init container image version to use type: string + init_container_extra_commands: + description: Extra commands for the init container + type: string + init_container_extra_volume_mounts: + description: Specify volume mounts to be added to the init container + type: string kind: description: Kind of the deployment type type: string diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 8a16f621..a9944ced 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -116,6 +116,22 @@ init_container_image_version: 8 image_pull_policy: IfNotPresent image_pull_secret: '' +# Extra commands which will be appended to the initContainer +# Make sure that each command entered return an exit code 0 +# otherwise the initContainer will fail +# init_container_extra_commands: | +# date >> /var/lib/awx/projects/timestamp +# chgrp 1000 /shared +# chmod 775 /shared +init_container_extra_commands: '' + +# Mount extra volumes on the initContainer. +# The volume used must be defined as an `extra_volumes` resource +# init_container_extra_volume_mounts: | +# - name: shared-vol +# mountPath: /shared +init_container_extra_volume_mounts: '' + ee_images: - name: AWX EE 0.5.0 image: quay.io/ansible/awx-ee:0.5.0 diff --git a/roles/installer/templates/deployment.yaml.j2 b/roles/installer/templates/deployment.yaml.j2 index f4cef315..a887c88f 100644 --- a/roles/installer/templates/deployment.yaml.j2 +++ b/roles/installer/templates/deployment.yaml.j2 @@ -33,7 +33,7 @@ spec: imagePullSecrets: - name: {{ image_pull_secret }} {% endif %} -{% if bundle_ca_crt %} +{% if bundle_ca_crt or projects_persistence|bool or init_container_extra_commands %} initContainers: - name: init image: '{{ init_container_image }}:{{ init_container_image_version }}' @@ -42,15 +42,33 @@ spec: - /bin/sh - -c - | +{% if bundle_ca_crt %} mkdir -p /etc/pki/ca-trust/extracted/{java,pem,openssl,edk2} update-ca-trust +{% endif %} +{% if projects_persistence|bool %} + chmod 775 /var/lib/awx/projects + chgrp 1000 /var/lib/awx/projects +{% endif %} +{% if init_container_extra_commands %} + {{ init_container_extra_commands | indent(width=14) }} +{% endif %} volumeMounts: +{% if bundle_ca_crt %} - name: "ca-trust-extracted" mountPath: "/etc/pki/ca-trust/extracted" - name: "{{ meta.name }}-bundle-cacert" mountPath: /etc/pki/ca-trust/source/anchors/bundle-ca.crt subPath: bundle-ca.crt readOnly: true +{% endif %} +{% if projects_persistence|bool %} + - name: "{{ meta.name }}-projects" + mountPath: "/var/lib/awx/projects" +{% endif %} +{% if init_container_extra_volume_mounts -%} + {{ init_container_extra_volume_mounts | indent(width=12, indentfirst=True) }} +{% endif %} {% endif %} containers: - image: '{{ redis_image }}:{{ redis_image_version }}' @@ -282,6 +300,10 @@ spec: {% if tolerations %} tolerations: {{ tolerations | indent(width=8) }} +{% endif %} +{% if projects_persistence|bool %} + securityContext: + fsGroup: 1000 {% endif %} volumes: {% if bundle_ca_crt %}