Merge pull request #1305 from john-westcott-iv/no_jmes

Modifying stage playbook to not need jmespath

.github/workflows/ci.yaml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   molecule:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     name: molecule
     env:
       DOCKER_API_VERSION: "1.38"
@@ -40,7 +40,7 @@ jobs:
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind
   helm:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     name: helm
     steps:
       - uses: actions/checkout@v2

.github/workflows/devel.yaml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   release:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     name: Push devel image
     steps:
       - uses: actions/checkout@v2

.github/workflows/feature.yml

@@ -0,0 +1,75 @@
+---
+
+name: Feature Branch Image Build and Push
+
+on:
+  push:
+    branches: [feature_*]
+
+jobs:
+  release:
+    runs-on: ubuntu-18.04
+    name: Push devel image
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # needed so that git describe --tag works
+
+      - name: Set VERSION
+        run: |
+          echo "VERSION=$(git describe --tags)" >>${GITHUB_ENV}
+
+      - name: Set lower case owner name
+        run: |
+          echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
+        env:
+          OWNER: '${{ github.repository_owner }}'
+
+      - name: Set IMAGE_TAG_BASE
+        run: |
+          echo "IMAGE_TAG_BASE=ghcr.io/${OWNER_LC}/awx-operator" >>${GITHUB_ENV}
+
+      - name: Set ARCH environment variable
+        run: |
+          echo "ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)" >>${GITHUB_ENV}
+
+      - name: Set OS environment variable
+        run: |
+          echo "OS=$(uname | awk '{print tolower($0)}')" >>${GITHUB_ENV}
+
+      - name: Install operator-sdk
+        run: |
+          echo "Installing operator-sdk ${OPERATOR_SDK_DL_URL}" && \
+          curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} && \
+          chmod +x operator-sdk_${OS}_${ARCH} && \
+          sudo mkdir -p /usr/local/bin/ && \
+          sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk && \
+          operator-sdk version
+        env:
+          OPERATOR_SDK_DL_URL: https://github.com/operator-framework/operator-sdk/releases/download/v1.26.0
+
+      - name: Log in to registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Build and Push awx-operator Image
+        run: |
+          make docker-build docker-push
+          docker tag ${IMAGE_TAG_BASE}:${VERSION} ${IMAGE_TAG_BASE}:${GITHUB_REF##*/}
+          docker push ${IMAGE_TAG_BASE}:${GITHUB_REF##*/}
+
+      - name: Build bundle manifests
+        run: |
+          make bundle
+
+      - name: Build and Push awx-operator Bundle
+        run: |
+          make bundle-build bundle-push
+          docker tag ${IMAGE_TAG_BASE}-bundle:v${VERSION} ${IMAGE_TAG_BASE}-bundle:${GITHUB_REF##*/}
+          docker push ${IMAGE_TAG_BASE}-bundle:${GITHUB_REF##*/}
+
+      - name: Build and Push awx-operator Catalog
+        run: |
+          make catalog-build catalog-push
+          docker tag ${IMAGE_TAG_BASE}-catalog:v${VERSION} ${IMAGE_TAG_BASE}-catalog:${GITHUB_REF##*/}
+          docker push ${IMAGE_TAG_BASE}-catalog:${GITHUB_REF##*/}

.github/workflows/promote.yaml

@@ -33,15 +33,11 @@ jobs:
           docker push quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
           docker push quay.io/${{ github.repository }}:latest
 
-      - name: Configure git
-        run: |
-          git config user.name "$GITHUB_ACTOR"
-          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-
       - name: Release Helm chart
         run: |
           ansible-playbook ansible/helm-release.yml -v \
             -e operator_image=quay.io/${{ github.repository }} \
             -e chart_owner=${{ github.repository_owner }} \
             -e tag=${{ github.event.release.tag_name }} \
-            -e gh_token=${{ secrets.GITHUB_TOKEN }}
+            -e gh_token=${{ secrets.GITHUB_TOKEN }} \
+            -e gh_user=${{ github.actor }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/operator-framework/ansible-operator:v1.25.3
+FROM quay.io/operator-framework/ansible-operator:v1.26.0
 
 USER 0
 

Makefile

@@ -156,7 +156,7 @@ ifeq (,$(shell which ansible-operator 2>/dev/null))
 	@{ \
 	set -e ;\
 	mkdir -p $(dir $(ANSIBLE_OPERATOR)) ;\
-	curl -sSLo $(ANSIBLE_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/v1.25.3/ansible-operator_$(OS)_$(ARCHA) ;\
+	curl -sSLo $(ANSIBLE_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/v1.26.0/ansible-operator_$(OS)_$(ARCHA) ;\
 	chmod +x $(ANSIBLE_OPERATOR) ;\
 	}
 else
@@ -187,7 +187,7 @@ ifeq (,$(shell which opm 2>/dev/null))
 	@{ \
 	set -e ;\
 	mkdir -p $(dir $(OPM)) ;\
-	curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.25.3/$(OS)-$(ARCHA)-opm ;\
+	curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.26.0/$(OS)-$(ARCHA)-opm ;\
 	chmod +x $(OPM) ;\
 	}
 else
@@ -395,9 +395,3 @@ helm-index:
 	$(HELM) repo index .cr-release-packages --url https://github.com/$(CHART_OWNER)/$(CHART_REPO)/releases/download/ --merge gh-pages/index.yaml
 
 	mv .cr-release-packages/index.yaml gh-pages/index.yaml
-
-	@echo "== PUSH INDEX FILE =="
-	cd gh-pages;\
-	git add index.yaml;\
-	git commit -m "Updated index.yaml latest release";\
-	git push;\

README.md

@@ -49,6 +49,7 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w
             * [Upgrade of instances without auto upgrade](#upgrade-of-instances-without-auto-upgrade)
          * [Service Account](#service-account)
          * [Labeling operator managed objects](#labeling-operator-managed-objects)
+         * [Pods termination grace period](#pods-termination-grace-period)
       * [Uninstall](#uninstall)
       * [Upgrading](#upgrading)
          * [Backup](#backup)
@@ -269,7 +270,7 @@ yDL2Cx5Za94g9MvBP6B73nzVLlmfgPjR
 
 You just completed the most basic install of an AWX instance via this operator. Congratulations!!!
 
-For an example using the Nginx Controller in Minukube, don't miss our [demo video](https://asciinema.org/a/416946).
+For an example using the Nginx Ingress Controller in Minikube, don't miss our [demo video](https://asciinema.org/a/416946).
 
 
 ### Helm Install on existing cluster
@@ -402,12 +403,14 @@ The following variables are customizable only when `service_type=LoadBalancer`
 | --------------------- | ---------------------------------------- | ------- |
 | loadbalancer_protocol | Protocol to use for Loadbalancer ingress | http    |
 | loadbalancer_port     | Port used for Loadbalancer ingress       | 80      |
+| loadbalancer_ip        | Assign Loadbalancer IP                   | ''      |  
 
 ```yaml
 ---
 spec:
   ...
   service_type: LoadBalancer
+  loadbalancer_ip: '192.168.10.25'
   loadbalancer_protocol: https
   loadbalancer_port: 443
   service_annotations: |
@@ -574,7 +577,7 @@ spec:
       cpu: 500m
       memory: 2Gi
     limits:
-      cpu: 1
+      cpu: '1'
       memory: 4Gi
   postgres_storage_requirements:
     requests:
@@ -869,7 +872,7 @@ A sample of extra settings can be found as below. All possible options can be fo
       value: 'LDAPSearch("OU=Groups,DC=abc,DC=com",ldap.SCOPE_SUBTREE,"(objectClass=group)",)'
 
     - setting: AUTH_LDAP_GROUP_TYPE
-      value: 'GroupOfNamesType(name_attr="cn")'
+      value: 'GroupOfNamesType'
 
     - setting: AUTH_LDAP_USER_ATTR_MAP
       value: '{"first_name": "givenName","last_name": "sn","email": "mail"}'
@@ -1246,6 +1249,46 @@ spec:
 ...
 ```
 
+#### Pods termination grace period
+
+During deployment restarts or new rollouts, when old ReplicaSet Pods are being
+terminated, the corresponding jobs which are managed (executed or controlled)
+by old AWX Pods may end up in `Error` state as there is no mechanism to
+transfer them to the newly spawned AWX Pods. To work around the problem one
+could set `termination_grace_period_seconds` in AWX spec, which does the
+following:
+
+* It sets the corresponding
+  [`terminationGracePeriodSeconds`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination)
+  Pod spec of the AWX Deployment to the value provided
+
+  > The grace period is the duration in seconds after the processes running in
+  > the pod are sent a termination signal and the time when the processes are
+  > forcibly halted with a kill signal
+
+* It adds a
+  [`PreStop`](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution)
+  hook script, which will keep AWX Pods in terminating state until it finished,
+  up to `terminationGracePeriodSeconds`.
+
+  > This grace period applies to the total time it takes for both the PreStop
+  > hook to execute and for the Container to stop normally
+
+  While the hook script just waits until the corresponding AWX Pod (instance)
+  no longer has any managed jobs, in which case it finishes with success and
+  hands over the overall Pod termination process to normal AWX processes.
+
+One may want to set this value to the maximum duration they accept to wait for
+the affected Jobs to finish. Keeping in mind that such finishing jobs may
+increase Pods termination time in such situations as `kubectl rollout restart`,
+AWX upgrade by the operator, or Kubernetes [API-initiated
+evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/).
+
+
+| Name                             | Description                                                     | Default |
+| -------------------------------- | --------------------------------------------------------------- | ------- |
+| termination_grace_period_seconds | Optional duration in seconds pods needs to terminate gracefully | not set |
+
 ### Uninstall ###
 
 To uninstall an AWX deployment instance, you basically need to remove the AWX kind related to that instance. For example, to delete an AWX instance named awx-demo, you would do:

ansible/helm-release.yml

@@ -14,34 +14,78 @@
           Release must exist before running this playbook
       when: release is not success
 
-    - name: Build and package helm chart
-      command: |
-        make helm-package
-      environment:
-        VERSION: "{{ tag }}"
-        IMAGE_TAG_BASE: "{{ operator_image }}"
-      args:
-        chdir: "{{ playbook_dir }}/../"
+    - name: Set helm filename and commit message
+      set_fact:
+        asset_already_attached: False
+        helm_file_name: "awx-operator-{{ tag }}.tgz"
+        commit_message: "Updated index.yaml for release {{ release.json.tag_name }}"
 
-    # Move to chart releaser after https://github.com/helm/chart-releaser/issues/122 exists
-    - name: Upload helm chart
-      uri:
-        url: "https://uploads.github.com/repos/{{ chart_owner }}/{{ chart_repo }}/releases/{{ release.json.id }}/assets?name=awx-operator-{{ tag }}.tgz"
-        src: "{{ playbook_dir }}/../.cr-release-packages/awx-operator-{{ tag }}.tgz"
-        headers:
-          Authorization: "token {{ gh_token }}"
-          Content-Type: "application/octet-stream"
-        status_code:
-          - 200
-          - 201
-      register: asset_upload
-      changed_when: asset_upload.json.state == "uploaded"
+    - name: See if file is already attached
+      set_fact:
+        asset_already_attached: True
+      loop: "{{ release.json.get('assets', []) }}"
+      loop_control:
+        label: "{{ item.name }}"
+      when: item.name == helm_file_name
 
-    - name: Publish helm index
-      command: |
-        make helm-index
-      environment:
-        CHART_OWNER: "{{ chart_owner }}"
-        CR_TOKEN: "{{ gh_token }}"
-      args:
-        chdir: "{{ playbook_dir }}/../"
+    - when: not asset_already_attached
+      block:
+        - name: Build and package helm chart
+          command: |
+            make helm-package
+          environment:
+            VERSION: "{{ tag }}"
+            IMAGE_TAG_BASE: "{{ operator_image }}"
+          args:
+            chdir: "{{ playbook_dir }}/../"
+
+        # Move to chart releaser after https://github.com/helm/chart-releaser/issues/122 exists
+        - name: Upload helm chart
+          uri:
+            url: "https://uploads.github.com/repos/{{ chart_owner }}/{{ chart_repo }}/releases/{{ release.json.id }}/assets?name={{ helm_file_name }}"
+            src: "{{ playbook_dir }}/../.cr-release-packages/awx-operator-{{ tag }}.tgz"
+            headers:
+              Authorization: "token {{ gh_token }}"
+              Content-Type: "application/octet-stream"
+            status_code:
+              - 200
+              - 201
+          register: asset_upload
+          changed_when: asset_upload.json.state == "uploaded"
+
+    - name: Ensure gh-pages exists
+      file:
+        state: directory
+        path: "{{ playbook_dir }}/../gh-pages"
+
+    - name: Check if we have published the release
+      command:
+        cmd: "git log --grep='{{ commit_message }}'"
+        chdir: "{{ playbook_dir }}/../gh-pages"
+      register: commits_for_release
+
+    - when: commits_for_release.stdout == ''
+      block:
+        - name: Configure git config
+          shell: |
+            git config user.name {{ gh_user }}
+            git config user.email {{ gh_user }}@users.noreply.github.com
+          args:
+            chdir: "{{ playbook_dir }}/../gh-pages"
+
+        - name: Publish helm index
+          command: |
+            make helm-index
+          environment:
+            CHART_OWNER: "{{ chart_owner }}"
+            CR_TOKEN: "{{ gh_token }}"
+          args:
+            chdir: "{{ playbook_dir }}/../"
+
+        - name: Stage and Push commit to gh-pages branch
+          shell: |
+            git add index.yaml
+            git commit -m "{{ commit_message }}"
+            git push
+          args:
+            chdir: "{{ playbook_dir }}/../gh-pages"

ansible/instantiate-awx-deployment.yml

@@ -26,6 +26,7 @@
             image_version: "{{ image_version | default(omit) }}"
             development_mode: "{{ development_mode | default(omit) | bool }}"
             image_pull_policy: "{{ image_pull_policy | default(omit) }}"
+            nodeport_port: "{{ nodeport_port | default(omit) }}"
             # ee_images:
             #   - name: test-ee
             #     image: quay.io/<user>/awx-ee

config/crd/bases/awx.ansible.com_awxrestores.yaml

@@ -39,12 +39,14 @@ spec:
           spec:
             type: object
             x-kubernetes-preserve-unknown-fields: true
+            required:
+            - deployment_name
             properties:
               backup_source:
                 description: Backup source
                 type: string
                 enum:
-                - CR
+                - Backup CR
                 - PVC
               deployment_name:
                 description: Name of the restored deployment. This should be different from the original deployment name

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -132,6 +132,10 @@ spec:
                 description: Port to use for the loadbalancer
                 type: integer
                 default: 80
+              loadbalancer_ip:
+                description: Assign LoadBalancer IP address
+                type: string
+                default: ''
               route_host:
                 description: The DNS to use to points to the instance
                 type: string
@@ -525,6 +529,10 @@ spec:
                         type: array
                     type: object
                 type: object
+              termination_grace_period_seconds:
+                description: Optional duration in seconds pods needs to terminate gracefully
+                type: integer
+                format: int32
               service_labels:
                 description: Additional labels to apply to the service
                 type: string
@@ -758,6 +766,9 @@ spec:
               init_container_extra_volume_mounts:
                 description: Specify volume mounts to be added to the init container
                 type: string
+              init_projects_container_image:
+                description: Registry path to the init projects container to use
+                type: string
               postgres_image:
                 description: Registry path to the PostgreSQL container to use
                 type: string

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -270,6 +270,12 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:number
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer
+      - displayName: LoadBalancer IP
+        path: loadbalancer_ip
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:string
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer
       - displayName: Route API Version
         path: route_api_version
         x-descriptors:
@@ -622,6 +628,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Termination Grace Period Seconds
+        path: termination_grace_period_seconds
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Service Labels
         path: service_labels
         x-descriptors:
@@ -687,6 +698,12 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: Init Projects Container image to use
+        displayName: Init Projects Container Image
+        path: init_projects_container_image
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - description: Secret where can be found the trusted Certificate Authority Bundle
         path: bundle_cacert_secret
         x-descriptors:

config/rbac/role.yaml

@@ -20,7 +20,6 @@ rules:
       - watch
   - apiGroups:
       - ""
-      - "rbac.authorization.k8s.io"
     resources:
       - pods
       - services
@@ -31,6 +30,17 @@ rules:
       - events
       - configmaps
       - secrets
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - "rbac.authorization.k8s.io"
+    resources:
       - roles
       - rolebindings
     verbs:
@@ -43,12 +53,22 @@ rules:
       - watch
   - apiGroups:
       - apps
-      - networking.k8s.io
     resources:
       - deployments
       - daemonsets
       - replicasets
       - statefulsets
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
       - ingresses
     verbs:
       - get

config/scorecard/patches/basic.config.yaml

@@ -4,7 +4,7 @@
     entrypoint:
     - scorecard-test
     - basic-check-spec
-    image: quay.io/operator-framework/scorecard-test:v1.25.3
+    image: quay.io/operator-framework/scorecard-test:v1.26.0
     labels:
       suite: basic
       test: basic-check-spec-test

config/scorecard/patches/olm.config.yaml

@@ -4,7 +4,7 @@
     entrypoint:
     - scorecard-test
     - olm-bundle-validation
-    image: quay.io/operator-framework/scorecard-test:v1.25.3
+    image: quay.io/operator-framework/scorecard-test:v1.26.0
     labels:
       suite: olm
       test: olm-bundle-validation-test
@@ -14,7 +14,7 @@
     entrypoint:
     - scorecard-test
     - olm-crds-have-validation
-    image: quay.io/operator-framework/scorecard-test:v1.25.3
+    image: quay.io/operator-framework/scorecard-test:v1.26.0
     labels:
       suite: olm
       test: olm-crds-have-validation-test
@@ -24,7 +24,7 @@
     entrypoint:
     - scorecard-test
     - olm-crds-have-resources
-    image: quay.io/operator-framework/scorecard-test:v1.25.3
+    image: quay.io/operator-framework/scorecard-test:v1.26.0
     labels:
       suite: olm
       test: olm-crds-have-resources-test
@@ -34,7 +34,7 @@
     entrypoint:
     - scorecard-test
     - olm-spec-descriptors
-    image: quay.io/operator-framework/scorecard-test:v1.25.3
+    image: quay.io/operator-framework/scorecard-test:v1.26.0
     labels:
       suite: olm
       test: olm-spec-descriptors-test
@@ -44,7 +44,7 @@
     entrypoint:
     - scorecard-test
     - olm-status-descriptors
-    image: quay.io/operator-framework/scorecard-test:v1.25.3
+    image: quay.io/operator-framework/scorecard-test:v1.26.0
     labels:
       suite: olm
       test: olm-status-descriptors-test

docs/debugging.md

@@ -1,4 +1,67 @@
-# Iterating on the installer without deploying the operator
+# Debugging the AWX Operator
+
+## General Debugging
+
+When the operator is deploying AWX, it is running the `installer` role inside the operator container. If the AWX CR's status is `Failed`, it is often useful to look at the awx-operator container logs, which shows the output of the installer role. To see these logs, run:
+
+```
+kubectl logs deployments/awx-operator-controller-manager -c awx-manager -f
+```
+
+### Inspect k8s Resources
+
+Past that, it is often useful to inspect various resources the AWX Operator manages like:
+* awx
+* awxbackup
+* awxrestore
+* pod
+* deployment
+* pvc
+* service
+* ingress
+* route
+* secrets
+* serviceaccount
+
+And if installing via OperatorHub and OLM:
+* subscription
+* csv
+* installPlan
+* catalogSource
+
+To inspect these resources you can use these commands
+
+```
+# Inspecting k8s resources
+kubectl describe -n <namespace> <resource> <resource-name>
+kubectl get -n <namespace> <resource> <resource-name> -o yaml
+kubectl logs -n <namespace> <resource> <resource-name>
+
+# Inspecting Pods
+kubectl exec -it -n <namespace> <pod> <pod-name>
+```
+
+
+### Configure No Log
+
+It is possible to show task output for debugging by setting no_log to false on the AWX CR spec.
+This will show output in the awx-operator logs for any failed tasks where no_log was set to true.
+
+For example:
+
+```
+---
+apiVersion: awx.ansible.com/v1beta1
+kind: AWX
+metadata:
+  name: awx-demo
+spec:
+  service_type: nodeport
+  no_log: false                  # <------------
+
+```
+
+## Iterating on the installer without deploying the operator
 
 Go through the [normal basic install](https://github.com/ansible/awx-operator/blob/devel/README.md#basic-install) steps.
 
@@ -35,6 +98,19 @@ ansible_operator_meta:
   namespace: awx
 service_type: nodeport
 ```
+The vars file will replace the awx resource so any value that you wish to over ride using the awx resource, put in the vars file. For example, if you wish to use your own image, version and pull policy, you can specify it like below:
+
+```yaml
+# vars.yml
+---
+ansible_operator_meta:
+  name: awx
+  namespace: awx
+service_type: nodeport
+image: $DEV_DOCKER_TAG_BASE/awx_kube_devel
+image_pull_policy: Always
+image_version: $COMPOSE_TAG
+```
 
 Run the installer:
 

docs/migration.md

@@ -34,7 +34,7 @@ metadata:
   namespace: <target namespace>
 stringData:
   host: <external ip or url resolvable by the cluster>
-  port: <external port, this usually defaults to 5432>
+  port: "<external port, this usually defaults to 5432>"    # quotes are required
   database: <desired database name>
   username: <username to connect as>
   password: <password to connect with>

roles/backup/tasks/creation.yml

@@ -32,22 +32,22 @@
   - this_backup['resources'][0]['metadata']['labels']
 
 - block:
-    - include_tasks: init.yml
+  - include_tasks: init.yml
 
-    - include_tasks: postgres.yml
+  - include_tasks: postgres.yml
 
-    - include_tasks: awx-cro.yml
+  - include_tasks: awx-cro.yml
 
-    - include_tasks: secrets.yml
+  - include_tasks: secrets.yml
 
-    - name: Set flag signifying this backup was successful
-      set_fact:
-        backup_complete: true
+  - name: Set flag signifying this backup was successful
+    set_fact:
+      backup_complete: true
 
-    - include_tasks: cleanup.yml
+  - include_tasks: cleanup.yml
 
   when:
-    - this_backup['resources'][0]['status']['backupDirectory'] is not defined
+  - this_backup['resources'][0]['status']['backupDirectory'] is not defined
 
 - name: Update status variables
   include_tasks: update_status.yml

roles/backup/tasks/postgres.yml

@@ -50,7 +50,7 @@
 
 - name: Determine the timestamp for the backup once for all nodes
   set_fact:
-    now: '{{ lookup("pipe", "date +%F-%T") }}'
+    now: '{{ lookup("pipe", "date +%F-%H%M%S") }}'
 
 - name: Set backup directory name
   set_fact:

roles/installer/files/pre-stop/termination-env

@@ -0,0 +1,66 @@
+# file, which when exists, indicates that `master` script has successfully
+# completed pre-stop script execution
+marker_file="${PRE_STOP_MARKER_FILE:-/var/lib/pre-stop/.termination_marker}"
+
+# file which the running `master` script continuously updates (mtime) to
+# indicate it's still running. this file is then read by `watcher`s to
+# understand if they still have to wait for `termination_marker`
+heartbeat_file="${PRE_STOP_HEARTBEAT_FILE:-/var/lib/pre-stop/.heartbeat}"
+
+# file which:
+#   * `watcher`s create when they bail out because they didn't see the
+#   `heartbeat_file` to be updated within `$heartbeat_failed_threshold`;
+#   * `master` creates when its handler command fails;
+# when scripts see such file, they also give up
+bailout_file="${PRE_STOP_BAILOUT_FILE:-/var/lib/pre-stop/.bailout}"
+heartbeat_threshold="${PRE_STOP_HEARTBEAT_THRESHOLD:-60}"
+
+# where the scripts' stdout/stderr are streamed
+stdout="${PRE_STOP_STDOUT:-/proc/1/fd/1}"
+stderr="${PRE_STOP_STDERR:-/proc/1/fd/2}"
+
+# command the `master` script executes, which when successfully finishes,
+# causes the script to create the `marker_file`
+handler="${PRE_STOP_HANDLER:-bash -c \"PYTHONUNBUFFERED=x awx-manage disable_instance --wait --retry=inf\"}"
+
+log_prefix="${PRE_STOP_LOG_PREFIX:-preStop.exec}"
+[[ -n ${PRE_STOP_LOG_ROLE} ]] && log_prefix="${log_prefix}] [$PRE_STOP_LOG_ROLE"
+
+# interval at which `watcher`s check for `marker_file` presence
+recheck_sleep="${PRE_STOP_RECHECK_SLEEP:-1}"
+# interval at which `watcher`s report into $stdout that they are still watching
+report_every="${PRE_STOP_REPORT_EVERY:-30}"
+
+function log {
+  printf "[%s] $1\n" "$log_prefix" "${@:2}"
+}
+
+function parameters_string {
+  for param in "$@"; do
+    printf "%s=\"%s\"\n" "$param" "${!param}"
+  done | paste -s -d ' '
+}
+
+function check_bailout {
+  if [[ -f $bailout_file ]]; then
+    log "\"%s\" file has been detected, accepting bail out signal and failing the hook script" \
+      "$bailout_file"
+    exit 1
+  fi
+}
+
+function check_heartbeat {
+  if [[ -f $heartbeat_file ]]; then
+    delta=$(( $(date +%s) - $(stat -c %Y "$heartbeat_file") ))
+  else
+    delta=$(( $(date +%s) - $1 ))
+  fi
+
+  if [[ $delta -gt $heartbeat_threshold ]]; then
+    log "The heartbeat file hasn't been updated since %ss, which is above the threshold of %ds, assuming the master is not operating and failing the hook script" \
+      $delta
+      $heartbeat_threshold
+    touch "$bailout_file"
+    exit 1
+  fi
+}

roles/installer/files/pre-stop/termination-master

@@ -0,0 +1,50 @@
+#/usr/bin/env bash
+
+PRE_STOP_LOG_ROLE="${PRE_STOP_LOG_ROLE:-master}"
+source $(dirname "$0")/termination-env
+
+{
+
+log "The hook has started: %s" \
+  "$(parameters_string \
+    "marker_file" \
+    "heartbeat_file" \
+    "bailout_file" \
+    "handler" \
+  )"
+
+touch "$heartbeat_file"
+
+set -o pipefail
+eval "$handler" 2>&1 | while IFS= read -r line; do
+  # we check the files here and break early, but overall script termination
+  # happens later - as we need to distinguish between files detection and
+  # command failure, while bash doesn't offer a simple way to do this here
+  # inside the loop (`exit` does not terminate the script)
+  [[ -f $bailout_file ]] && break
+  [[ -f $marker_file ]] && break
+
+  log "[handler] %s" "$line"
+  touch "$heartbeat_file"
+done
+ec=$?
+set +o pipefail
+
+# process various cases in specific order
+check_bailout
+
+if [[ -f $marker_file ]]; then
+  log "Done! The marker file has been detected, assuming some other instance of the script has run to completion"
+  exit 0
+elif [[ $ec -ne 0 ]]; then
+  log "The handler has failed with \"%d\" exit code, failing the hook script too" \
+    $ec
+  # signal others to bail out
+  touch "$bailout_file"
+  exit $ec
+else
+  log "Done! Generating the marker file allowing to proceed to termination"
+  touch "$marker_file"
+fi
+
+} > "$stdout" 2> "$stderr"

roles/installer/files/pre-stop/termination-waiter

@@ -0,0 +1,33 @@
+#/usr/bin/env bash
+
+PRE_STOP_LOG_ROLE="${PRE_STOP_LOG_ROLE:-waiter}"
+source $(dirname "$0")/termination-env
+
+{
+
+log "The hook has started: %s" \
+  "$(parameters_string \
+    "marker_file" \
+    "heartbeat_file" \
+    "bailout_file" \
+    "recheck_sleep" \
+    "report_every" \
+  )"
+
+n=0
+checks_started=$(date +%s)
+
+while ! [[ -f $marker_file ]]; do
+  check_bailout
+  check_heartbeat $checks_started
+
+  if [[ $(($n % $report_every)) -eq 0 ]]; then
+    log "Waiting for the marker file to be accessible..."
+  fi
+  n=$(($n + 1))
+  sleep $recheck_sleep
+done
+
+log "The marker file found, exiting to proceed to termination"
+
+} > "$stdout" 2> "$stderr"

roles/installer/tasks/install.yml

@@ -39,17 +39,17 @@
 - name: Load LDAP CAcert certificate
   include_tasks: load_ldap_cacert_secret.yml
   when:
-    - ldap_cacert_secret != ''
+  - ldap_cacert_secret != ''
 
 - name: Load ldap bind password
   include_tasks: load_ldap_password_secret.yml
   when:
-    - ldap_password_secret != ''
+  - ldap_password_secret != ''
 
 - name: Load bundle certificate authority certificate
   include_tasks: load_bundle_cacert_secret.yml
   when:
-    - bundle_cacert_secret != ''
+  - bundle_cacert_secret != ''
 
 - name: Include admin password configuration tasks
   include_tasks: admin_password_configuration.yml
@@ -66,8 +66,8 @@
 - name: Load Route TLS certificate
   include_tasks: load_route_tls_secret.yml
   when:
-    - ingress_type | lower == 'route'
-    - route_tls_secret != ''
+  - ingress_type | lower == 'route'
+  - route_tls_secret != ''
 
 - name: Include resources configuration tasks
   include_tasks: resources_configuration.yml
@@ -91,8 +91,8 @@
       bash -c "awx-manage migrate --noinput"
   register: migrate_result
   when:
-    - database_check is defined
-    - (database_check.stdout|trim) != '0'
+  - database_check is defined
+  - (database_check.stdout|trim) != '0'
 
 - name: Initialize Django
   include_tasks: initialize_django.yml

roles/installer/tasks/resources_configuration.yml

@@ -13,9 +13,17 @@
       - status.phase=Running
   register: tower_pod
 
+- name: Set the resource pod as a variable.
+  set_fact:
+    tower_pod: >-
+      {{ tower_pod['resources']
+         | rejectattr('metadata.deletionTimestamp', 'defined')
+         | sort(attribute='metadata.creationTimestamp')
+         | first | default({}) }}
+
 - name: Set the resource pod name as a variable.
   set_fact:
-    tower_pod_name: "{{ tower_pod['resources'][0]['metadata']['name'] | default('') }}"
+    tower_pod_name: "{{ tower_pod['metadata']['name'] | default('') }}"
 
 - name: Set user provided control plane ee image
   set_fact:
@@ -32,13 +40,13 @@
     kind: Secret
     namespace: '{{ ansible_operator_meta.namespace }}'
     name: '{{ ansible_operator_meta.name }}-receptor-ca'
-  register: _receptor_ca
+  register: receptor_ca
   no_log: "{{ no_log }}"
 
 - name: Migrate Receptor CA Secret
   when:
-    - _receptor_ca['resources'] | default([]) | length
-    - _receptor_ca['resources'][0]['type'] != "kubernetes.io/tls"
+    - receptor_ca['resources'] | default([]) | length
+    - receptor_ca['resources'][0]['type'] != "kubernetes.io/tls"
   block:
     - name: Delete old Receptor CA Secret
       k8s:
@@ -53,7 +61,7 @@
       register: _receptor_ca_key_file
     - name: Copy Receptor CA key from old secret to tempfile
       copy:
-        content: "{{ _receptor_ca['resources'][0]['data']['receptor-ca.key'] | b64decode }}"
+        content: "{{ receptor_ca['resources'][0]['data']['receptor-ca.key'] | b64decode }}"
         dest: "{{ _receptor_ca_key_file.path }}"
       no_log: "{{ no_log }}"
     - name: Create tempfile for receptor-ca.crt
@@ -63,7 +71,7 @@
       register: _receptor_ca_crt_file
     - name: Copy Receptor CA cert from old secret to tempfile
       copy:
-        content: "{{ _receptor_ca['resources'][0]['data']['receptor-ca.crt'] | b64decode }}"
+        content: "{{ receptor_ca['resources'][0]['data']['receptor-ca.crt'] | b64decode }}"
         dest: "{{ _receptor_ca_crt_file.path }}"
       no_log: "{{ no_log }}"
     - name: Create New Receptor CA secret
@@ -71,6 +79,17 @@
         apply: true
         definition: "{{ lookup('template', 'secrets/receptor_ca_secret.yaml.j2') }}"
       no_log: "{{ no_log }}"
+    - name: Read New Receptor CA Secret
+      k8s_info:
+        kind: Secret
+        namespace: '{{ ansible_operator_meta.namespace }}'
+        name: '{{ ansible_operator_meta.name }}-receptor-ca'
+      register: _receptor_ca
+      no_log: "{{ no_log }}"
+    - name: Set receptor_ca variable
+      set_fact:
+        receptor_ca: '{{ _receptor_ca }}'
+      no_log: "{{ no_log }}"
     - name: Remove tempfiles
       file:
         path: "{{ item }}"
@@ -106,6 +125,17 @@
         apply: true
         definition: "{{ lookup('template', 'secrets/receptor_ca_secret.yaml.j2') }}"
       no_log: "{{ no_log }}"
+    - name: Read Receptor CA secret
+      k8s_info:
+        kind: Secret
+        namespace: '{{ ansible_operator_meta.namespace }}'
+        name: '{{ ansible_operator_meta.name }}-receptor-ca'
+      register: _receptor_ca
+      no_log: "{{ no_log }}"
+    - name: Set receptor_ca variable
+      set_fact:
+        receptor_ca: '{{ _receptor_ca }}'
+      no_log: "{{ no_log }}"
     - name: Remove tempfiles
       file:
         path: "{{ item }}"
@@ -113,14 +143,14 @@
       loop:
         - "{{ _receptor_ca_key_file.path }}"
         - "{{ _receptor_ca_crt_file.path }}"
-  when: not _receptor_ca['resources'] | default([]) | length
+  when: not receptor_ca['resources'] | default([]) | length
 
 - name: Check for Receptor work signing Secret
   k8s_info:
     kind: Secret
     namespace: '{{ ansible_operator_meta.namespace }}'
     name: '{{ ansible_operator_meta.name }}-receptor-work-signing'
-  register: _receptor_work_signing
+  register: receptor_work_signing
   no_log: "{{ no_log }}"
 
 - name: Generate Receptor work signing RSA key pair
@@ -151,6 +181,17 @@
         apply: true
         definition: "{{ lookup('template', 'secrets/receptor_work_signing_secret.yaml.j2') }}"
       no_log: "{{ no_log }}"
+    - name: Read Receptor work signing Secret
+      k8s_info:
+        kind: Secret
+        namespace: '{{ ansible_operator_meta.namespace }}'
+        name: '{{ ansible_operator_meta.name }}-receptor-work-signing'
+      register: _receptor_work_signing
+      no_log: "{{ no_log }}"
+    - name: Set receptor_work_signing variable
+      set_fact:
+        receptor_work_signing: '{{ _receptor_work_signing }}'
+      no_log: "{{ no_log }}"
     - name: Remove tempfiles
       file:
         path: "{{ item }}"
@@ -158,16 +199,16 @@
       loop:
         - "{{ _receptor_work_signing_private_key_file.path }}"
         - "{{ _receptor_work_signing_public_key_file.path }}"
-  when: not _receptor_work_signing['resources'] | default([]) | length
+  when: not receptor_work_signing['resources'] | default([]) | length
 
 - name: Apply Resources
   k8s:
     apply: yes
     definition: "{{ lookup('template', item + '.yaml.j2') }}"
     wait: yes
-  register: tower_resources_result
   loop:
     - 'configmaps/config'
+    - 'configmaps/pre_stop_scripts'
     - 'secrets/app_credentials'
     - 'rbac/service_account'
     - 'storage/persistent'
@@ -210,21 +251,10 @@
     apply: yes
     definition: "{{ lookup('template', 'deployments/deployment.yaml.j2') }}"
     wait: yes
+    wait_timeout: "{{ 120 * replicas or 120 }}"
   register: this_deployment_result
 
 - block:
-    - name: Delete pod to reload a resource configuration
-      k8s:
-        api_version: v1
-        state: absent
-        kind: Pod
-        namespace: '{{ ansible_operator_meta.namespace }}'
-        name: '{{ tower_pod_name }}'
-        wait: yes
-      when:
-        - tower_resources_result.changed
-        - tower_pod_name | length
-
     - name: Get the new resource pod information after updating resource.
       k8s_info:
         kind: Pod
@@ -236,17 +266,20 @@
         field_selectors:
           - status.phase=Running
       register: _new_pod
-      until:
-        - _new_pod['resources'] | length
-        - _new_pod['resources'][0]['metadata']['name'] != tower_pod_name
-      delay: 5
-      retries: 60
+
+    - name: Update new resource pod as a variable.
+      set_fact:
+        tower_pod: >-
+          {{ _new_pod['resources']
+             | rejectattr('metadata.deletionTimestamp', 'defined')
+             | sort(attribute='metadata.creationTimestamp')
+             | last | default({}) }}
 
     - name: Update new resource pod name as a variable.
       set_fact:
-        tower_pod_name: '{{ _new_pod["resources"][0]["metadata"]["name"] }}'
+        tower_pod_name: '{{ tower_pod["metadata"]["name"] | default("")}}'
   when:
-    - tower_resources_result.changed or this_deployment_result.changed
+    - this_deployment_result.changed
 
 - name: Verify the resource pod name is populated.
   assert:

roles/installer/tasks/secret_key_configuration.yml

@@ -40,10 +40,10 @@
 
 - name: Set secret key secret
   set_fact:
-    __secret_key_secret: '{{ _generated_secret_key["resources"] | default([]) | length | ternary(_generated_secret_key, _secret_key_secret)  }}'
+    secret_key: '{{ _generated_secret_key["resources"] | default([]) | length | ternary(_generated_secret_key, _secret_key_secret)  }}'
   no_log: "{{ no_log }}"
 
 - name: Store secret key secret name
   set_fact:
-    secret_key_secret_name: "{{ __secret_key_secret['resources'][0]['metadata']['name'] }}"
+    secret_key_secret_name: "{{ secret_key['resources'][0]['metadata']['name'] }}"
   no_log: "{{ no_log }}"

roles/installer/tasks/set_images.yml

@@ -17,3 +17,20 @@
       {{ _custom_init_container_image |
          default(lookup('env', 'RELATED_IMAGE_AWX_INIT_CONTAINER')) |
          default(_default_init_container_image, true) }}
+
+- name: Set default awx init projects container image
+  set_fact:
+    _default_init_projects_container_image: "{{ _init_projects_container_image }}"
+
+- name: Set user provided awx init projects image
+  set_fact:
+    _custom_init_projects_container_image: "{{ init_projects_container_image }}"
+  when:
+    - init_projects_container_image | default([]) | length
+
+- name: Set Init projects image URL
+  set_fact:
+    _init_projects_container_image: >-
+      {{ _custom_init_projects_container_image |
+         default(lookup('env', 'RELATED_IMAGE_AWX_INIT_PROJECTS_CONTAINER')) |
+         default(_default_init_projects_container_image, true) }}

roles/installer/templates/configmaps/config.yaml.j2

@@ -100,6 +100,7 @@ data:
         include       /etc/nginx/mime.types;
         default_type  application/octet-stream;
         server_tokens off;
+        client_max_body_size 5M;
     
         log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                           '$status $body_bytes_sent "$http_referer" '

roles/installer/templates/configmaps/pre_stop_scripts.yaml.j2

@@ -0,0 +1,16 @@
+{% if termination_grace_period_seconds is defined %}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-pre-stop-scripts'
+  namespace: '{{ ansible_operator_meta.namespace }}'
+  labels:
+    {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }}
+data:
+  termination-master: |
+    {{ lookup("file", "files/pre-stop/termination-master") | indent(width=4) | trim }}
+  termination-waiter: |
+    {{ lookup("file", "files/pre-stop/termination-waiter") | indent(width=4) | trim }}
+  termination-env: |
+    {{ lookup("file", "files/pre-stop/termination-env") | indent(width=4) | trim }}
+{% endif %}

roles/installer/templates/deployments/deployment.yaml.j2

@@ -20,8 +20,26 @@ spec:
       labels:
         {{ lookup("template", "../common/templates/labels/common.yaml.j2")  | indent(width=8) | trim }}
         {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=8) | trim }}
-{% if annotations %}
       annotations:
+{% for template in [
+    "configmaps/config",
+    "configmaps/pre_stop_scripts",
+    "secrets/app_credentials",
+    "storage/persistent",
+  ] %}
+        checksum-{{ template | replace('/', '-') }}: "{{ lookup('template', template + '.yaml.j2') | sha1 }}"
+{% endfor %}
+{% for secret in [
+    "bundle_cacert",
+    "route_tls",
+    "ldap_cacert",
+    "secret_key",
+    "receptor_ca",
+    "receptor_work_signing",
+  ] %}
+        checksum-secret-{{ secret }}: "{{ lookup('ansible.builtin.vars', secret, default='')["resources"][0]["data"] | default('') | sha1 }}"
+{% endfor %}
+{% if annotations %}
         {{ annotations | indent(width=8) }}
 {% endif %}
     spec:
@@ -122,6 +140,23 @@ spec:
               mountPath: "/var/run/redis"
             - name: "{{ ansible_operator_meta.name }}-redis-data"
               mountPath: "/data"
+{% if termination_grace_period_seconds is defined %}
+            - name: pre-stop-data
+              mountPath: /var/lib/pre-stop
+            - name: pre-stop-scripts
+              mountPath: /var/lib/pre-stop/scripts
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - bash
+                - -c
+                # redis image doesn't support writing to `/proc/1/fd/*`
+                - >
+                  PRE_STOP_STDOUT=/dev/stdout
+                  PRE_STOP_STDERR=/dev/stderr
+                  /var/lib/pre-stop/scripts/termination-waiter
+{% endif %}
           resources: {{ redis_resource_requirements }}
         - image: '{{ _image }}'
           name: '{{ ansible_operator_meta.name }}-web'
@@ -291,6 +326,18 @@ spec:
 {% endif %}
 {% if task_extra_volume_mounts -%}
             {{ task_extra_volume_mounts | indent(width=12, first=True) }}
+{% endif %}
+{% if termination_grace_period_seconds is defined %}
+            - name: pre-stop-data
+              mountPath: /var/lib/pre-stop
+            - name: pre-stop-scripts
+              mountPath: /var/lib/pre-stop/scripts
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - bash
+                - /var/lib/pre-stop/scripts/termination-master
 {% endif %}
           env:
             - name: SUPERVISOR_WEB_CONFIG_PATH
@@ -360,6 +407,18 @@ spec:
               mountPath: "/var/lib/awx/projects"
 {% if ee_extra_volume_mounts -%}
             {{ ee_extra_volume_mounts | indent(width=12, first=True) }}
+{% endif %}
+{% if termination_grace_period_seconds is defined %}
+            - name: pre-stop-data
+              mountPath: /var/lib/pre-stop
+            - name: pre-stop-scripts
+              mountPath: /var/lib/pre-stop/scripts
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - bash
+                - /var/lib/pre-stop/scripts/termination-waiter
 {% endif %}
           env:
 {% if development_mode | bool %}
@@ -395,6 +454,9 @@ spec:
 {% if security_context_settings|length %}
         {{ security_context_settings | to_nice_yaml | indent(8) }}
 {% endif %}
+{% endif %}
+{% if termination_grace_period_seconds is defined %}
+      terminationGracePeriodSeconds: {{ termination_grace_period_seconds }}
 {% endif %}
       volumes:
 {% if bundle_ca_crt %}
@@ -424,6 +486,14 @@ spec:
             items:
               - key: ldap-ca.crt
                 path: 'ldap-ca.crt'
+{% endif %}
+{% if termination_grace_period_seconds is defined %}
+        - name: pre-stop-data
+          emptyDir: {}
+        - name: pre-stop-scripts
+          configMap:
+            name: '{{ ansible_operator_meta.name }}-{{ deployment_type }}-pre-stop-scripts'
+            defaultMode: 0775
 {% endif %}
         - name: "{{ ansible_operator_meta.name }}-application-credentials"
           secret:

roles/installer/templates/networking/service.yaml.j2

@@ -53,6 +53,9 @@ spec:
   type: NodePort
 {% elif service_type | lower == "loadbalancer" %}
   type: LoadBalancer
+{% if variable is defined and variable|length %}
+  loadbalancerip: '{{ loadbalancer_ip }}'
+{% endif %}
 {% else %}
   type: ClusterIP
 {% endif %}

roles/restore/tasks/import_vars.yml

@@ -2,24 +2,19 @@
 
 - name: Import awx_object variables
   block:
-  - name: Get AWX object definition from pvc
-    k8s_exec:
-      namespace: "{{ backup_pvc_namespace }}"
-      pod: "{{ ansible_operator_meta.name }}-db-management"
-      command: >-
-        bash -c "cat '{{ backup_dir }}/awx_object'"
-    register: awx_object
 
   - name: Create temp file for spec dict
     tempfile:
       state: file
     register: tmp_spec
 
-  - name: Write spec vars to temp file
-    copy:
-      content: "{{ awx_object.stdout }}"
-      dest: "{{ tmp_spec.path }}"
-      mode: '0644'
+  - name: Get AWX object definition from pvc
+    k8s_cp:
+      namespace: "{{ backup_pvc_namespace }}"
+      pod: "{{ ansible_operator_meta.name }}-db-management"
+      remote_path: "{{ backup_dir  }}/awx_object"
+      local_path: "{{ tmp_spec.path }}"
+      state: from_pod
 
   - name: Include spec vars to save them as a dict
     include_vars: "{{ tmp_spec.path }}"

roles/restore/tasks/main.yml

@@ -32,24 +32,24 @@
   - this_restore['resources'][0]['metadata']['labels']
 
 - block:
-    - include_tasks: init.yml
+  - include_tasks: init.yml
 
-    - include_tasks: import_vars.yml
+  - include_tasks: import_vars.yml
 
-    - include_tasks: secrets.yml
+  - include_tasks: secrets.yml
 
-    - include_tasks: deploy_awx.yml
+  - include_tasks: deploy_awx.yml
 
-    - include_tasks: postgres.yml
+  - include_tasks: postgres.yml
 
-    - name: Set flag signifying this restore was successful
-      set_fact:
-        tower_restore_complete: True
+  - name: Set flag signifying this restore was successful
+    set_fact:
+      tower_restore_complete: True
 
-    - include_tasks: cleanup.yml
+  - include_tasks: cleanup.yml
 
   when:
-    - this_restore['resources'][0]['status']['restoreComplete'] is not defined
+  - this_restore['resources'][0]['status']['restoreComplete'] is not defined
 
 - name: Update status variables
   include_tasks: update_status.yml

roles/restore/tasks/postgres.yml

@@ -66,9 +66,8 @@
 
 - name: Set full resolvable host name for postgres pod
   set_fact:
-    resolvable_db_host: "{{ awx_postgres_host }}.{{ ansible_operator_meta.namespace }}.svc.{{ cluster_name }}"
+    resolvable_db_host: '{{ (awx_postgres_type == "managed") | ternary(awx_postgres_host + "." + ansible_operator_meta.namespace + ".svc." + cluster_name, awx_postgres_host) }}'  # yamllint disable-line rule:line-length
   no_log: "{{ no_log }}"
-  when: awx_postgres_type == 'managed'
 
 - name: Set pg_restore command
   set_fact:

roles/restore/tasks/secrets.yml

@@ -1,25 +1,18 @@
 ---
 
-- name: Get secret definition from pvc
-  k8s_exec:
-    namespace: "{{ backup_pvc_namespace }}"
-    pod: "{{ ansible_operator_meta.name }}-db-management"
-    command: >-
-      bash -c "cat '{{ backup_dir }}/secrets.yml'"
-  register: _secrets
-  no_log: "{{ no_log }}"
-
 - name: Create Temporary secrets file
   tempfile:
     state: file
     suffix: .json
   register: tmp_secrets
 
-- name: Write vars to file locally
-  copy:
-    dest: "{{ tmp_secrets.path }}"
-    content: "{{ _secrets.stdout }}"
-    mode: 0640
+- name: Get secret definition from pvc
+  k8s_cp:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ ansible_operator_meta.name }}-db-management"
+    remote_path: "{{ backup_dir  }}/secrets.yml"
+    local_path: "{{ tmp_secrets.path }}"
+    state: from_pod
   no_log: "{{ no_log }}"
 
 - name: Include secret vars from backup