Add support for horizontal pod autoscaling (#1676)

* feat: implement extra_settings_files
* fix: reduce duplicated code blocks by templates
* docs: update docs for extra settings
* docs: simplify the commands
* docs: add notes for duplicated keys in setting files

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/docs"
+    groups:
+      dependencies:
+        patterns:
+          - "*"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "component:docs"
+      - "dependencies"

.github/workflows/ci.yaml

@@ -17,6 +17,7 @@ jobs:
           - -t replicas
     env:
       DOCKER_API_VERSION: "1.41"
+      DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
     steps:
       - uses: actions/checkout@v3
 
@@ -37,10 +38,18 @@ jobs:
           MOLECULE_VERBOSITY: 3
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          STORE_DEBUG_OUTPUT: true
         run: |
           sudo rm -f $(which kustomize)
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }}
+
+      - name: Upload artifacts for failed tests if Run Molecule fails
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: awx_operator_molecule_test
+          path: ${{ env.DEBUG_OUTPUT_DIR }}
   helm:
     runs-on: ubuntu-latest
     name: helm
@@ -99,3 +108,5 @@ jobs:
             echo "${no_log}"
             exit 1
           fi
+  nox-sessions:
+    uses: ./.github/workflows/reusable-nox.yml

.github/workflows/devel.yaml

@@ -13,15 +13,36 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Build Image
+      - name: Fail if QUAY_REGISTRY not set
         run: |
-          IMG=awx-operator:devel make docker-build
+          if [[ -z "${{ vars.QUAY_REGISTRY }}" ]]; then
+            echo "QUAY_REGISTRY not set. Please set QUAY_REGISTRY in variable GitHub Actions variables."
+            exit 1
+          fi
 
-      - name: Push To Quay
-        uses: redhat-actions/push-to-registry@v2.1.1
+      - name: Log into registry ghcr.io
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
         with:
-          image: awx-operator
-          tags: devel
-          registry: quay.io/ansible/
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+
+      - name: Log into registry quay.io
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
+        with:
+          registry: ${{ vars.QUAY_REGISTRY }}
           username: ${{ secrets.QUAY_USER }}
           password: ${{ secrets.QUAY_TOKEN }}
+
+
+      - name: Build and Store Image @ghcr
+        run: |
+          IMG=ghcr.io/${{ github.repository }}:${{ github.sha }} make docker-buildx
+
+
+      - name: Publish Image to quay.io
+        run: |
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository }}:${{ github.sha }} \
+            --tag ${{ vars.QUAY_REGISTRY }}/awx-operator:devel

.github/workflows/feature.yml

@@ -29,25 +29,6 @@ jobs:
         run: |
           echo "IMAGE_TAG_BASE=ghcr.io/${OWNER_LC}/awx-operator" >>${GITHUB_ENV}
 
-      - name: Set ARCH environment variable
-        run: |
-          echo "ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)" >>${GITHUB_ENV}
-
-      - name: Set OS environment variable
-        run: |
-          echo "OS=$(uname | awk '{print tolower($0)}')" >>${GITHUB_ENV}
-
-      - name: Install operator-sdk
-        run: |
-          echo "Installing operator-sdk ${OPERATOR_SDK_DL_URL}" && \
-          curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} && \
-          chmod +x operator-sdk_${OS}_${ARCH} && \
-          sudo mkdir -p /usr/local/bin/ && \
-          sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk && \
-          operator-sdk version
-        env:
-          OPERATOR_SDK_DL_URL: https://github.com/operator-framework/operator-sdk/releases/download/v1.26.0
-
       - name: Log in to registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

.github/workflows/promote.yaml

@@ -3,37 +3,81 @@ name: Promote AWX Operator image
 on:
   release:
     types: [published]
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Name for the tag of the release.'
+        required: true
+      quay_registry:
+        description: 'Quay registry to push to.'
+        default: 'quay.io/ansible'
+
+env:
+  QUAY_REGISTRY: ${{ vars.QUAY_REGISTRY }}
 
 jobs:
   promote:
     runs-on: ubuntu-latest
     steps:
+      - name: Set GitHub Env vars for workflow_dispatch event
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          echo "TAG_NAME=${{ github.event.inputs.tag_name }}" >> $GITHUB_ENV
+          echo "QUAY_REGISTRY=${{ github.event.inputs.quay_registry }}" >> $GITHUB_ENV
+
+      - name: Set GitHub Env vars if release event
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          echo "TAG_NAME=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
+
+      - name: Fail if QUAY_REGISTRY not set
+        run: |
+          if [[ -z "${{ env.QUAY_REGISTRY }}" ]]; then
+            echo "QUAY_REGISTRY not set. Please set QUAY_REGISTRY in variable GitHub Actions variables."
+            exit 1
+          fi
+
       - uses: actions/checkout@v3
         with:
           depth: 0
 
-      - name: Log in to GHCR
-        run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
-      - name: Log in to Quay
-        run: |
-          echo ${{ secrets.QUAY_TOKEN }} | docker login quay.io -u ${{ secrets.QUAY_USER }} --password-stdin
+      - name: Log into registry ghcr.io
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Re-tag and promote awx-operator image
+
+      - name: Log into registry quay.io
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
+        with:
+          registry: ${{ env.QUAY_REGISTRY }}
+          username: ${{ secrets.QUAY_USER }}
+          password: ${{ secrets.QUAY_TOKEN }}
+
+
+      - name: Pull Tagged Staged Image and Publish to quay.io
         run: |
-          docker pull ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:latest
-          docker push quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker push quay.io/${{ github.repository }}:latest
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} \
+            --tag ${{ env.QUAY_REGISTRY }}/awx-operator:${{ env.TAG_NAME }}
+
+
+      - name: Pull Staged Image and Publish to quay.io/${{ github.repository }}:latest
+        run: |
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} \
+            --tag ${{ env.QUAY_REGISTRY }}/awx-operator:latest
+
 
       - name: Release Helm chart
         run: |
           ansible-playbook ansible/helm-release.yml -v \
-            -e operator_image=quay.io/${{ github.repository }} \
+            -e operator_image=${{ env.QUAY_REGISTRY }}/awx-operator \
             -e chart_owner=${{ github.repository_owner }} \
-            -e tag=${{ github.event.release.tag_name }} \
+            -e tag=${{ env.TAG_NAME }} \
             -e gh_token=${{ secrets.GITHUB_TOKEN }} \
             -e gh_user=${{ github.actor }} \
             -e repo_type=https

.github/workflows/publish-operator-hub.yaml

@@ -0,0 +1,86 @@
+name: Publish AWX Operator on operator-hub
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Name for the tag of the release.'
+        required: true
+      operator_hub_fork:
+        description: 'Fork of operator-hub where the PR will be created from. default: awx-auto'
+        required: true
+        default: 'awx-auto'
+      image_registry:
+        description: 'Image registry where the image is published to. default: quay.io'
+        required: true
+        default: 'quay.io'
+      image_registry_organization:
+        description: 'Image registry organization where the image is published to. default: ansible'
+        required: true
+        default: 'ansible'
+      community_operator_github_org:
+        description: 'Github organization for community-opeartor project. default: k8s-operatorhub'
+        required: true
+        default: 'k8s-operatorhub'
+      community_operator_prod_github_org:
+        description: 'GitHub organization for community-operator-prod project. default: redhat-openshift-ecosystem'
+        required: true
+        default: 'redhat-openshift-ecosystem'
+jobs:
+  promote:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set GITHUB_ENV from workflow_dispatch event
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          echo "VERSION=${{ github.event.inputs.tag_name }}" >> $GITHUB_ENV
+          echo "IMAGE_REGISTRY=${{ github.event.inputs.image_registry }}" >> $GITHUB_ENV
+          echo "IMAGE_REGISTRY_ORGANIZATION=${{ github.event.inputs.image_registry_organization }}" >> $GITHUB_ENV
+          echo "COMMUNITY_OPERATOR_GITHUB_ORG=${{ github.event.inputs.community_operator_github_org }}" >> $GITHUB_ENV
+          echo "COMMUNITY_OPERATOR_PROD_GITHUB_ORG=${{ github.event.inputs.community_operator_prod_github_org }}" >> $GITHUB_ENV
+
+      - name: Set GITHUB_ENV for release event
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
+          echo "IMAGE_REGISTRY=quay.io" >> $GITHUB_ENV
+          echo "IMAGE_REGISTRY_ORGANIZATION=ansible" >> $GITHUB_ENV
+          echo "COMMUNITY_OPERATOR_GITHUB_ORG=k8s-operatorhub" >> $GITHUB_ENV
+          echo "COMMUNITY_OPERATOR_PROD_GITHUB_ORG=redhat-openshift-ecosystem" >> $GITHUB_ENV
+
+      - name: Log in to image registry
+        run: |
+          echo ${{ secrets.QUAY_TOKEN }} | docker login ${{ env.IMAGE_REGISTRY }} -u ${{ secrets.QUAY_USER }} --password-stdin
+
+      - name: Checkout awx-operator at workflow branch
+        uses: actions/checkout@v4
+        with:
+          path: awx-operator
+
+      - name: Checkout awx-opearator at ${{ env.VERSION }}
+        uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          ref: ${{ env.VERSION }}
+          path: awx-operator-${{ env.VERSION }}
+          fetch-depth: 0  # fetch all history so that git describe works
+
+      - name: Copy scripts to awx-operator-${{ env.VERSION }}
+        run: |
+          cp -f \
+            awx-operator/hack/publish-to-operator-hub.sh \
+            awx-operator-${{ env.VERSION }}/hack/publish-to-operator-hub.sh
+          cp -f \
+            awx-operator/Makefile \
+            awx-operator-${{ env.VERSION }}/Makefile
+
+      - name: Build and publish bundle to operator-hub
+        working-directory: awx-operator-${{ env.VERSION }}
+        env:
+          IMG_REPOSITORY: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_REGISTRY_ORGANIZATION }}
+          GITHUB_TOKEN: ${{ secrets.AWX_AUTO_GITHUB_TOKEN }}
+        run: |
+          git config --global user.email "awx-automation@redhat.com"
+          git config --global user.name "AWX Automation"
+          ./hack/publish-to-operator-hub.sh

.github/workflows/reusable-nox.yml

@@ -0,0 +1,26 @@
+---
+name: nox
+
+"on":
+    workflow_call:
+
+jobs:
+    nox:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                - session: build
+                  python-versions: "3.11"
+        name: "Run nox ${{ matrix.session }} session"
+        steps:
+        - name: Check out repo
+          uses: actions/checkout@v4
+        - name: Setup nox
+          uses: wntrblm/nox@2024.03.02
+          with:
+              python-versions: "${{ matrix.python-versions }}"
+        - name: "Run nox -s ${{ matrix.session }}"
+          run: |
+            nox -s "${{ matrix.session }}"

.github/workflows/stage.yml

@@ -37,12 +37,6 @@ jobs:
 
           exit 0
 
-      - name: Checkout awx
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ github.repository_owner }}/awx
-          path: awx
-
       - name: Checkout awx-operator
         uses: actions/checkout@v3
         with:
@@ -53,17 +47,20 @@ jobs:
         run: |
           python3 -m pip install docker
 
-      - name: Log in to GHCR
-        run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Log into registry ghcr.io
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and stage awx-operator
+      - name: Stage awx-operator
         working-directory: awx-operator
         run: |
           BUILD_ARGS="--build-arg DEFAULT_AWX_VERSION=${{ github.event.inputs.default_awx_version }} \
-                      --build-arg OPERATOR_VERSION=${{ github.event.inputs.version }}" \
-          IMAGE_TAG_BASE=ghcr.io/${{ github.repository_owner }}/awx-operator \
-          VERSION=${{ github.event.inputs.version }} make docker-build docker-push
+              --build-arg OPERATOR_VERSION=${{ github.event.inputs.version }}" \
+          IMG=ghcr.io/${{ github.repository }}:${{ github.event.inputs.version }} \
+          make docker-buildx
 
       - name: Run test deployment
         working-directory: awx-operator
@@ -76,10 +73,12 @@ jobs:
         env:
           AWX_TEST_VERSION: ${{ github.event.inputs.default_awx_version }}
 
-      - name: Create draft release
-        working-directory: awx
-        run: |
-          ansible-playbook tools/ansible/stage.yml \
-            -e version=${{ github.event.inputs.version }} \
-            -e repo=${{ github.repository_owner }}/awx-operator \
-            -e github_token=${{ secrets.GITHUB_TOKEN }}
+      - name: Create Draft Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.event.inputs.version }}
+          release_name: Release ${{ github.event.inputs.version }}
+          draft: true

.gitignore

@@ -10,3 +10,4 @@ gh-pages/
 .vscode/
 __pycache__
 /site
+venv/*

.helm/starter/README.md

@@ -8,10 +8,12 @@ To configure your AWX resource using this chart, create your own `yaml` values f
 In your values config, enable `AWX.enabled` and add `AWX.spec` values based on the awx operator's [documentation](https://github.com/ansible/awx-operator/blob/devel/README.md). Consult the docs below for additional functionality.
 
 ### Installing
-The operator's [helm install](https://github.com/ansible/awx-operator/blob/devel/README.md#helm-install-on-existing-cluster) guide provides key installation instructions.
+
+The operator's [helm install](https://ansible.readthedocs.io/projects/awx-operator/en/latest/installation/helm-install-on-existing-cluster.html) guide provides key installation instructions.
 
 Example:
-```
+
+```bash
 helm install my-awx-operator awx-operator/awx-operator -n awx --create-namespace -f myvalues.yaml
 ```
 
@@ -24,6 +26,39 @@ Argument breakdown:
 
 To update an existing installation, use `helm upgrade` instead of `install`. The rest of the syntax remains the same.
 
+### Caveats on upgrading existing installation
+
+There is no support at this time for upgrading or deleting CRDs using Helm.  See [helm documentation](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) for additional detail.
+
+When upgrading to releases with CRD changes use the following command to update the CRDs
+
+```bash
+kubectl apply --server-side -k github.com/ansible/awx-operator/config/crd?ref=<VERSION>
+```
+
+If running above command results in an error like below:
+
+```text
+Apply failed with 1 conflict: conflict with "helm" using apiextensions.k8s.io/v1: .spec.versions
+Please review the fields above--they currently have other managers. Here
+are the ways you can resolve this warning:
+* If you intend to manage all of these fields, please re-run the apply
+  command with the `--force-conflicts` flag.
+* If you do not intend to manage all of the fields, please edit your
+  manifest to remove references to the fields that should keep their
+  current managers.
+* You may co-own fields by updating your manifest to match the existing
+  value; in this case, you'll become the manager if the other manager(s)
+  stop managing the field (remove it from their configuration).
+See https://kubernetes.io/docs/reference/using-api/server-side-apply/#conflicts
+```
+
+Use `--force-conflicts` flag to resolve the conflict.
+
+```bash
+kubectl apply --server-side --force-conflicts -k github.com/ansible/awx-operator/config/crd?ref=<VERSION>
+```
+
 ## Configuration
 The goal of adding helm configurations is to abstract out and simplify the creation of multi-resource configs. The `AWX.spec` field maps directly to the spec configs of the `AWX` resource that the operator provides, which are detailed in the [main README](https://github.com/ansible/awx-operator/blob/devel/README.md). Other sub-config can be added with the goal of simplifying more involved setups that require additional resources to be specified.
 
@@ -32,6 +67,158 @@ These sub-headers aim to be a more intuitive entrypoint into customizing your de
 ### External Postgres
 The `AWX.postgres` section simplifies the creation of the external postgres secret. If enabled, the configs provided will automatically be placed in a `postgres-config` secret and linked to the `AWX` resource. For proper secret management, the `AWX.postgres.password` value, and any other sensitive values, can be passed in at the command line rather than specified in code. Use the `--set` argument with `helm install`. Supplying the password this way is not recommended for production use, but may be helpful for initial PoC.
 
+### Additional Kubernetes Resources
+The `AWX.extraDeploy` section allows the creation of additional Kubernetes resources. This simplifies setups requiring additional objects that are used by AWX, e.g. using `ExternalSecrets` to create Kubernetes secrets.
+
+Resources are passed as an array, either as YAML or strings (literal "|"). The resources are passed through `tpl`, so templating is possible. Example:
+
+```yaml
+AWX:
+  # enable use of awx-deploy template
+  ...
+
+  # configurations for external postgres instance
+  postgres:
+    enabled: false
+    ...
+
+extraDeploy:
+  - |
+    apiVersion: external-secrets.io/v1beta1
+    kind: ExternalSecret
+    metadata:
+      name: {{ .Release.Name }}-postgres-secret-string-example
+      namespace: {{ .Release.Namespace }}
+      labels:
+        app: {{ .Release.Name }}
+    spec:
+      secretStoreRef:
+        name: vault
+        kind: ClusterSecretStore
+      refreshInterval: "1h"
+      target:
+        name: postgres-configuration-secret-string-example
+        creationPolicy: "Owner"
+        deletionPolicy: "Delete"
+      dataFrom:
+        - extract:
+            key: awx/postgres-configuration-secret
+
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ExternalSecret
+    metadata:
+      name: "{{ .Release.Name }}-postgres-secret-yaml-example"
+      namespace: "{{ .Release.Namespace }}"
+      labels:
+        app: "{{ .Release.Name }}"
+    spec:
+      secretStoreRef:
+        name: vault
+        kind: ClusterSecretStore
+      refreshInterval: "1h"
+      target:
+        name: postgres-configuration-secret-yaml-example
+        creationPolicy: "Owner"
+        deletionPolicy: "Delete"
+      dataFrom:
+        - extract:
+            key: awx/postgres-configuration-secret
+```
+
+### Custom secrets
+The `customSecrets` section simplifies the creation of our custom secrets used during AWX deployment. Supplying the passwords this way is not recommended for production use, but may be helpful for initial PoC.
+
+If enabled, the configs provided will automatically used to create the respective secrets and linked at the CR spec level. For proper secret management, the sensitive values can be passed in at the command line rather than specified in code. Use the `--set` argument with `helm install`.
+
+Example:
+
+```yaml
+AWX:
+  # enable use of awx-deploy template
+  ...
+
+  # configurations for external postgres instance
+  postgres:
+    enabled: false
+    ...
+
+  customSecrets:
+    enabled: true
+    admin:
+      enabled: true
+      password: mysuperlongpassword
+      secretName: my-admin-password
+    secretKey:
+      enabled: true
+      key: supersecuresecretkey
+      secretName: my-awx-secret-key
+    ingressTls:
+      enabled: true
+      selfSignedCert: true
+      key: unset
+      certificate: unset
+    routeTls:
+      enabled: false
+      key: <contentoftheprivatekey>
+      certificate: <contentofthepublickey>
+    ldapCacert:
+      enabled: false
+      crt: <contentofmybundlecacrt>
+    ldap:
+      enabled: true
+      password: yourldapdnpassword
+    bundleCacert:
+      enabled: false
+      crt: <contentofmybundlecacrt>
+    eePullCredentials:
+      enabled: false
+      url: unset
+      username: unset
+      password: unset
+      sslVerify: true
+      secretName: my-ee-pull-credentials
+    cpPullCredentials:
+      enabled: false
+      dockerconfig:
+        - registry: https://index.docker.io/v1/
+          username: unset
+          password: unset
+      secretName: my-cp-pull-credentials
+```
+
+### Custom volumes
+The `customVolumes` section simplifies the creation of Persistent Volumes used when you want to store your databases and projects files on the cluster's Node. Since their backends are `hostPath`, the size specified are just like a label and there is no actual capacity limitation.
+
+You have to prepare directories for these volumes. For example:
+
+```bash
+sudo mkdir -p /data/postgres-13
+sudo mkdir -p /data/projects
+sudo chmod 755 /data/postgres-13
+sudo chown 1000:0 /data/projects
+```
+
+Example:
+
+```yaml
+AWX:
+  # enable use of awx-deploy template
+  ...
+
+  # configurations for external postgres instance
+  postgres:
+    enabled: false
+    ...
+
+  customVolumes:
+    postgres:
+      enabled: true
+      hostPath: /data/postgres-13
+    projects:
+      enabled: true
+      hostPath: /data/projects
+      size: 1Gi
+```
 
 ## Values Summary
 
@@ -43,6 +230,120 @@ The `AWX.postgres` section simplifies the creation of the external postgres secr
 | `AWX.spec` | specs to directly configure the AWX resource | `{}` |
 | `AWX.postgres` | configurations for the external postgres secret | - |
 
+### extraDeploy
+| Value | Description | Default |
+|---|---|---|
+| `extraDeploy` | array of additional resources to be deployed (supports YAML or literal "\|") | - |
+
+### customSecrets
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.enabled` | Enable the secret resources configuration | `false` |
+| `customSecrets.admin` | Configurations for the secret that contains the admin user password | - |
+| `customSecrets.secretKey` | Configurations for the secret that contains the symmetric key for encryption | - |
+| `customSecrets.ingressTls` | Configurations for the secret that contains the TLS information when `ingress_type=ingress` | - |
+| `customSecrets.routeTls` |  Configurations for the secret that contains the TLS information when `ingress_type=route` (`route_tls_secret`) | - |
+| `customSecrets.ldapCacert` | Configurations for the secret that contains the LDAP Certificate Authority | - |
+| `customSecrets.ldap` | Configurations for the secret that contains the LDAP BIND DN password | - |
+| `customSecrets.bundleCacert` | Configurations for the secret that contains the Certificate Authority | - |
+| `customSecrets.eePullCredentials` | Configurations for the secret that contains the pull credentials for registered ees can be found | - |
+| `customSecrets.cpPullCredentials` | Configurations for the secret that contains the image pull credentials for app and database containers | - |
+
+
+Below the addition variables to customize the secret configuration.
+
+#### Admin user password secret configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.admin.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.admin.password` | Admin user password | - |
+| `customSecrets.admin.secretName` | Name of secret for `admin_password_secret` | `<resourcename>-admin-password>` |
+
+#### Secret Key secret configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.secretKey.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.secretKey.key` | Key is used to encrypt sensitive data in the database | - |
+| `customSecrets.secretKey.secretName` | Name of secret for `secret_key_secret` | `<resourcename>-secret-key` |
+
+#### Ingress TLS secret configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.ingressTls.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.ingressTls.selfSignedCert` | If `true`, an self-signed TLS certificate for `AWX.spec.hostname` will be create by helm | `false` |
+| `customSecrets.ingressTls.key` | Private key to use for TLS/SSL | - |
+| `customSecrets.ingressTls.certificate` | Certificate to use for TLS/SSL | - |
+| `customSecrets.ingressTls.secretName` | Name of secret for `ingress_tls_secret` | `<resourcename>-ingress-tls` |
+| `customSecrets.ingressTls.labels` | Array of labels for the secret | - |
+
+#### Route TLS secret configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.routeTls.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.routeTls.key` | Private key to use for TLS/SSL | - |
+| `customSecrets.routeTls.certificate` | Certificate to use for TLS/SSL | - |
+| `customSecrets.routeTls.secretName` | Name of secret for `route_tls_secret` | `<resourcename>-route-tls` |
+
+#### LDAP Certificate Authority secret configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.ldapCacert.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.ldapCacert.crt` | Bundle of CA Root Certificates | - |
+| `customSecrets.ldapCacert.secretName` | Name of secret for `ldap_cacert_secret` | `<resourcename>-custom-certs` |
+
+#### LDAP BIND DN Password secret configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.ldap.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.ldap.password` | LDAP BIND DN password | - |
+| `customSecrets.ldap.secretName` | Name of secret for `ldap_password_secret` | `<resourcename>-ldap-password` |
+
+#### Certificate Authority secret configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.bundleCacert.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.bundleCacert.crt` | Bundle of CA Root Certificates | - |
+| `customSecrets.bundleCacert.secretName` | Name of secret for `bundle_cacert_secret` | `<resourcename>-custom-certs` |
+
+#### Default EE pull secrets configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.eePullCredentials.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.eePullCredentials.url` | Registry url | - |
+| `customSecrets.eePullCredentials.username` | Username to connect as | - |
+| `customSecrets.eePullCredentials.password` | Password to connect with | - |
+| `customSecrets.eePullCredentials.sslVerify` | Whether verify ssl connection or not. | `true` |
+| `customSecrets.eePullCredentials.secretName` | Name of secret for `ee_pull_credentials_secret` | `<resourcename>-ee-pull-credentials` |
+
+#### Control Plane pull secrets configuration
+| Value | Description | Default |
+|---|---|---|
+| `customSecrets.cpPullCredentials.enabled` | If `true`, secret will be created | `false` |
+| `customSecrets.cpPullCredentials.dockerconfig` | Array of configurations for the Docker credentials that are used for accessing a registry | - |
+| `customSecrets.cpPullCredentials.dockerconfig[].registry` | Server location for Docker registry | `https://index.docker.io/v1/` |
+| `customSecrets.cpPullCredentials.dockerconfig[].username` | Username to connect as | - |
+| `customSecrets.cpPullCredentials.dockerconfig[].password` | Password to connect with | - |
+| `customSecrets.cpPullCredentials.secretName` |  Name of secret for `image_pull_secrets`| `<resoucename>-cp-pull-credentials` |
+
+### customVolumes
+
+#### Persistent Volume for databases postgres
+| Value | Description | Default |
+|---|---|---|
+| `customVolumes.postgres.enabled` | Enable the PV resource configuration for the postgres databases | `false` |
+| `customVolumes.postgres.hostPath` | Directory location on host | - |
+| `customVolumes.postgres.size` | Size of the volume | `8Gi` |
+| `customVolumes.postgres.accessModes` | Volume access mode | `ReadWriteOnce` |
+| `customVolumes.postgres.storageClassName` | PersistentVolume storage class name for `postgres_storage_class` | `<resourcename>-postgres-volume` |
+
+#### Persistent Volume for projects files
+| Value | Description | Default |
+|---|---|---|
+| `customVolumes.projects.enabled` | Enable the PVC and PVC resources configuration for the projects files | `false` |
+| `customVolumes.projects.hostPath` | Directory location on host | - |
+| `customVolumes.projects.size` |  Size of the volume | `8Gi` |
+| `customVolumes.projects.accessModes` | Volume access mode | `ReadWriteOnce` |
+| `customVolumes.postgres.storageClassName` | PersistentVolume storage class name | `<resourcename>-projects-volume` |
 
 # Contributing
 
@@ -63,5 +364,3 @@ The chart is currently hosted on the gh-pages branch of the repo. During the rel
 Instead of CR, we use `helm repo index` to generate an index from all locally pulled chart versions. Since we build from scratch every time, the timestamps of all entries will be updated. This could be improved by using yq or something similar to detect which tags are already in the index.yaml file, and only merge in tags that are not present.
 
 Not using CR could be addressed in the future by keeping the chart built as a part of releases, as long as CR compares the chart to previous release packages rather than previous commits. If the latter is the case, then we would not have the necessary history for comparison.
-
-

.helm/starter/templates/awx-deploy.yaml

@@ -6,6 +6,10 @@ metadata:
   name: {{ .name }}
   namespace: {{ $.Release.Namespace }}
 spec:
+{{- /* Provide custom persistent volumes configs if enabled */}}
+{{- include "spec.storageClassNames" $ }}
+{{- /* Provide custom secrets configs if enabled */}}
+{{- include "spec.secrets" $ }}
 {{- /* Include raw map from the values file spec */}}
 {{ .spec | toYaml | indent 2 }}
 {{- /* Provide security context defaults */}}

.helm/starter/templates/extra-list.yaml

@@ -0,0 +1,8 @@
+{{- range .Values.extraDeploy }}
+---
+  {{- if typeIs "string" . }}
+    {{- tpl . $ }}
+  {{- else }}
+    {{- tpl (. | toYaml | nindent 0) $ }}
+  {{- end }}
+{{- end }}

.helm/starter/templates/postgres-config.yaml

@@ -12,6 +12,7 @@ stringData:
   username: {{ .username }}
   password: {{ .password }}
   sslmode: {{ .sslmode }}
+  target_session_attrs: {{ .target_session_attrs | default "any" }}
   type: {{ .type }}
 type: Opaque
 {{- end }}

.helm/starter/templates/secrets/_helpers.tpl

@@ -0,0 +1,170 @@
+{{/*
+Generate certificates for ingress
+*/}}
+{{- define "ingress.gen-certs" -}}
+{{- $ca := genCA "ingress-ca" 365 -}}
+{{- $cert := genSignedCert ( $.Values.AWX.spec.hostname | required "AWX.spec.hostname is required!" ) nil nil 365 $ca -}}
+tls.crt: {{ $cert.Cert | b64enc }}
+tls.key: {{ $cert.Key | b64enc }}
+{{- end -}}
+
+{{/*
+Generate the name of the secret that contains the admin user password
+*/}}
+{{- define "admin.secretName" -}}
+{{ default (printf "%s-admin-password" $.Values.AWX.name) (default $.Values.customSecrets.admin.secretName $.Values.AWX.spec.admin_password_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the TLS information when ingress_type=route
+*/}}
+{{- define "routeTls.secretName" -}}
+{{ default (printf "%s-route-tls" $.Values.AWX.name) (default $.Values.customSecrets.routeTls.secretName $.Values.AWX.spec.route_tls_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the TLS information when ingress_type=ingress
+*/}}
+{{- define "ingressTls.secretName" -}}
+{{ default (printf "%s-ingress-tls" $.Values.AWX.name) (default $.Values.customSecrets.ingressTls.secretName $.Values.AWX.spec.ingress_tls_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the LDAP Certificate Authority
+*/}}
+{{- define "ldapCacert.secretName" -}}
+{{ default (printf "%s-custom-certs" $.Values.AWX.name) (default ($.Values.customSecrets.ldapCacert).secretName $.Values.AWX.spec.ldap_cacert_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the custom Certificate Authority
+*/}}
+{{- define "bundleCacert.secretName" -}}
+{{ default (printf "%s-custom-certs" $.Values.AWX.name) (default ($.Values.customSecrets.bundleCacert).secretName $.Values.AWX.spec.bundle_cacert_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the LDAP BIND DN password
+*/}}
+{{- define "ldap.secretName" -}}
+{{ default (printf "%s-ldap-password" $.Values.AWX.name) (default $.Values.customSecrets.ldap.secretName $.Values.AWX.spec.ldap_password_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the symmetric key for encryption
+*/}}
+{{- define "secretKey.secretName" -}}
+{{ default (printf "%s-secret-key" $.Values.AWX.name) (default $.Values.customSecrets.secretKey.secretName $.Values.AWX.spec.secret_key_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the default execution environment pull credentials
+*/}}
+{{- define "eePullCredentials.secretName" -}}
+{{ default (printf "%s-ee-pull-credentials" $.Values.AWX.name) (default $.Values.customSecrets.eePullCredentials.secretName $.Values.AWX.spec.ee_pull_credentials_secret) }}
+{{- end }}
+
+{{/*
+Generate the name of the secret that contains the default control plane pull credentials
+*/}}
+{{- define "cpPullCredentials.secretName" -}}
+{{ default (printf "%s-cp-pull-credentials" $.Values.AWX.name) (default $.Values.customSecrets.cpPullCredentials.secretName $.Values.AWX.spec.image_pull_secrets) }}
+{{- end }}
+
+{{/*
+Generate the .dockerconfigjson file unencoded.
+*/}}
+{{- define "dockerconfigjson.b64dec" }}
+  {{- print "{\"auths\":{" }}
+  {{- range $index, $item := . }}
+    {{- if $index }}
+      {{- print "," }}
+    {{- end }}
+    {{- printf "\"%s\":{\"username\":\"%s\",\"password\":\"%s\",\"email\":\"%s\",\"auth\":\"%s\"}" (default "https://index.docker.io/v1/" $item.registry) $item.username $item.password (default "" $item.email) (printf "%s:%s" $item.username $item.password | b64enc) }}
+  {{- end }}
+  {{- print "}}" }}
+{{- end }}
+
+{{/*
+Generate the base64-encoded .dockerconfigjson.
+*/}}
+{{- define "dockerconfigjson.b64enc" }}
+  {{- $list := ternary (list .) . (kindIs "map" .) }}
+  {{- include "dockerconfigjson.required" $list }}
+  {{- include "dockerconfigjson.b64dec" $list | b64enc }}
+{{- end }}
+
+{{/*
+Required values for .dockerconfigjson
+*/}}
+{{- define "dockerconfigjson.required" -}}
+  {{- range . -}}
+    {{- $_ := required "cpPullCredentials.dockerconfigjson[].username is required!" .username -}}
+    {{- $_ := required "cpPullCredentials.dockerconfigjson[].password is required!" .password -}}
+  {{- end -}}
+  {{/* Check for registry uniqueness */}}
+  {{- $registries := list -}}
+  {{- range . -}}
+    {{- $registries = append $registries (default "https://index.docker.io/v1/" .registry) -}}
+  {{- end -}}
+  {{- $_ := required "All cpPullCredentials.dockerconfigjson[].registry's must be unique!" (or (eq (len $registries) (len ($registries | uniq))) nil) -}}
+{{- end -}}
+
+{{/*
+Generate the name of the secrets
+*/}}
+{{- define "spec.secrets" -}}
+{{- /* secret configs if enabled */}}
+{{- if hasKey $.Values "customSecrets" }}
+{{- with $.Values.customSecrets }}
+{{- if .enabled }}
+  {{- if hasKey . "admin" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "admin_password_secret")) .admin.enabled }}
+  admin_password_secret: {{ include "admin.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "secretKey" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "secret_key_secret")) .secretKey.enabled }}
+  secret_key_secret: {{ include "secretKey.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "routeTls" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "route_tls_secret")) .routeTls.enabled }}
+  route_tls_secret: {{ include "routeTls.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "ingressTls" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "ingress_tls_secret")) .ingressTls.enabled }}
+  ingress_tls_secret: {{ include "ingressTls.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "ldapCacert" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "ldap_cacert_secret")) .ldapCacert.enabled }}
+  ldap_cacert_secret: {{ include "ldapCacert.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "bundleCacert" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "bundle_cacert_secret")) .bundleCacert.enabled }}
+  bundle_cacert_secret: {{ include "bundleCacert.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "ldap" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "ldap_password_secret")) .ldap.enabled }}
+  ldap_password_secret: {{ include "ldap.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "eePullCredentials" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "ee_pull_credentials_secret")) .eePullCredentials.enabled }}
+  ee_pull_credentials_secret: {{ include "eePullCredentials.secretName" $ }}
+  {{- end }}
+  {{- end }}
+  {{- if hasKey . "cpPullCredentials" }}
+  {{- if and (not (hasKey $.Values.AWX.spec "image_pull_secrets")) .cpPullCredentials.enabled }}
+  image_pull_secrets:
+    - {{ include "cpPullCredentials.secretName" $ }}
+  {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/admin-password-secret.yaml

@@ -0,0 +1,16 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- if hasKey .Values.customSecrets "admin" }}
+{{- with $.Values.customSecrets.admin }}
+{{- if .enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "admin.secretName" $ }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+data:
+  password: {{ .password | required "customSecrets.admin.password is required!" | b64enc }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/cp-pull-credentials-secret.yaml

@@ -0,0 +1,16 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- if hasKey .Values.customSecrets "cpPullCredentials" }}
+{{- with $.Values.customSecrets.cpPullCredentials }}
+{{- if .enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "cpPullCredentials.secretName" $ }}
+  namespace: {{ $.Release.Namespace }}
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: {{ template "dockerconfigjson.b64enc" .dockerconfig | required "customSecrets.cpPullCredentials.dockerconfig is required!" }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/custom-certs-secret.yaml

@@ -0,0 +1,49 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- with .Values.customSecrets }}
+{{- $hasLdapCrt := (hasKey $.Values.customSecrets "ldapCacert") -}}
+{{- $hasBundleCrt := (hasKey . "bundleCacert") -}}
+{{- if or $hasLdapCrt $hasBundleCrt }}
+{{- $ldapCrtEnabled := ternary (.ldapCacert).enabled false $hasLdapCrt -}}
+{{- $bundleCrtEnabled := ternary (.bundleCacert).enabled false $hasBundleCrt -}}
+{{- $ldapSecretName := (include "ldapCacert.secretName" $) -}}
+{{- $bundleSecretName := (include "bundleCacert.secretName" $) -}}
+{{- if and (or $bundleCrtEnabled $ldapCrtEnabled) (eq $ldapSecretName $bundleSecretName) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $ldapSecretName }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+data:
+{{- if $ldapCrtEnabled }}
+  ldap-ca.crt: {{ .ldapCacert.crt | required "customSecrets.ldapCacert.crt is required!" | b64enc }}
+{{- end }}
+{{- if $bundleCrtEnabled }}
+  bundle-ca.crt: {{ .bundleCacert.crt | required "customSecrets.bundleCacert.crt is required!" | b64enc }}
+{{- end }}
+{{- else }}
+{{- if $ldapCrtEnabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $ldapSecretName }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+data:
+  ldap-ca.crt: {{ .ldapCacert.crt | required "customSecrets.ldapCacert.crt is required!" | b64enc }}
+{{- end }}
+{{- if $bundleCrtEnabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $bundleSecretName }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+data:
+  bundle-ca.crt: {{ .bundleCacert.crt | required "customSecrets.bundleCacert.crt is required!" | b64enc }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/ee-pull-credentials-secret.yaml

@@ -0,0 +1,19 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- if hasKey .Values.customSecrets "eePullCredentials" }}
+{{- with $.Values.customSecrets.eePullCredentials }}
+{{- if .enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "eePullCredentials.secretName" $ }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+stringData:
+  url: {{ (required "customSecrets.eePullCredentials.url is required!" .url) | quote }}
+  username: {{ (required "customSecrets.eePullCredentials.username is required!" .username) | quote }}
+  password: {{ (required "customSecrets.eePullCredentials.password is required!" .password) | quote }}
+  ssl_verify: {{ or .sslVerify (eq (.sslVerify | toString) "<nil>") | quote }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/ingress-tls-secret.yaml

@@ -0,0 +1,25 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- if hasKey .Values.customSecrets "ingressTls" }}
+{{- with $.Values.customSecrets.ingressTls }}
+{{- if .enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "ingressTls.secretName" $ }}
+  namespace: {{ $.Release.Namespace }}
+{{- if .labels }}
+  labels:
+{{ toYaml .labels | indent 4 }}
+{{- end }}
+type: kubernetes.io/tls
+data:
+{{- if .selfSignedCert }}
+{{ ( include "ingress.gen-certs" $ ) | indent 2 }}
+{{ else }}
+  tls.key: {{ (.key | required "customSecrets.ingressTls.key is required!") | b64enc }}
+  tls.crt: {{ (.certificate | required "customSecrets.ingressTls.certificate is required!") | b64enc }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/ldap-password-secret.yaml

@@ -0,0 +1,16 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- if hasKey .Values.customSecrets "ldap" }}
+{{- with $.Values.customSecrets.ldap }}
+{{- if .enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "ldap.secretName" $ }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+data:
+  ldap-password: {{ .password | required "customSecrets.ldap.password is required!" | b64enc }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/route-tls-secret.yaml

@@ -0,0 +1,17 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- if hasKey .Values.customSecrets "routeTls" }}
+{{- with $.Values.customSecrets.routeTls }}
+{{- if .enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "routeTls.secretName" $ }}
+  namespace: {{ $.Release.Namespace }}
+type: kubernetes.io/tls
+data:
+  tls.key: {{ (.key | required "customSecrets.routeTls.key is required!") | b64enc }}
+  tls.crt: {{ (.certificate | required "customSecrets.routeTls.certificate is required!") | b64enc }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/secrets/secret-key-secret.yaml

@@ -0,0 +1,16 @@
+{{- if ($.Values.customSecrets).enabled }}
+{{- if hasKey .Values.customSecrets "secretKey" }}
+{{- with $.Values.customSecrets.secretKey }}
+{{- if .enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "secretKey.secretName" $ }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+stringData:
+  secret_key: {{ .key | required "customSecrets.secretKey.key is required!" | quote }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/storage/_helpers.tpl

@@ -0,0 +1,57 @@
+{{/*
+Generate the name of the persistent volume for postgres folders
+*/}}
+{{- define "postgres.persistentVolumeName" -}}
+{{ printf "%s-postgres-volume" $.Values.AWX.name }}
+{{- end }}
+
+{{/*
+Generate the name of the persistent volume for projects folder
+*/}}
+{{- define "projects.persistentVolumeName" -}}
+{{ printf "%s-projects-volume" $.Values.AWX.name }}
+{{- end }}
+
+{{/*
+Generate the name of the persistent volume claim for the projects volume
+*/}}
+{{- define "projects.persistentVolumeClaim" -}}
+{{ printf "%s-projects-claim" $.Values.AWX.name }}
+{{- end }}
+
+{{/*
+Generate the name of the storage class to use for the postgres volume
+*/}}
+{{- define "postgres.storageClassName" -}}
+{{ default (printf "%s-postgres-volume" $.Values.AWX.name) (default $.Values.AWX.spec.postgres_storage_class (($.Values.customVolumes).postgres).storageClassName) }}
+{{- end }}
+
+{{/*
+Generate the name of the storage class to use for the projects volume
+*/}}
+{{- define "projects.storageClassName" -}}
+{{ default (printf "%s-projects-volume" $.Values.AWX.name) (default $.Values.AWX.spec.projects_storage_class (($.Values.customVolumes).projects).storageClassName) }}
+{{- end }}
+
+{{/*
+Generate the name of the storage class names, expects AWX context passed in
+*/}}
+{{- define "spec.storageClassNames" -}}
+{{- if and (not $.Values.AWX.postgres.enabled) (eq (($.Values.AWX.spec).postgres_configuration_secret | default "") "") -}}
+{{- if (($.Values.customVolumes).postgres).enabled -}}
+  {{- if not (hasKey $.Values.AWX.spec "postgres_storage_class") }}
+  postgres_storage_class: {{ include "postgres.storageClassName" $ }}    
+  {{- end }}
+  {{- if not (hasKey $.Values.AWX.spec "postgres_storage_requirements") }}
+  postgres_storage_requirements:
+    requests:
+      storage: {{ default "8Gi" $.Values.customVolumes.postgres.size | quote }}
+  {{- end }}
+{{- end }}
+{{- end }}
+{{- if and ($.Values.AWX.spec.projects_persistence) (eq (($.Values.AWX.spec).projects_existing_claim | default "") "") -}}
+{{- if (($.Values.customVolumes).projects).enabled }}
+  projects_existing_claim: {{ include "projects.persistentVolumeClaim" $ }} 
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/storage/postgres-pv.yaml

@@ -0,0 +1,19 @@
+{{- if and (not $.Values.AWX.postgres.enabled) (eq (($.Values.AWX.spec).postgres_configuration_secret | default "") "") -}}
+{{- if (($.Values.customVolumes).postgres).enabled -}}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "postgres.persistentVolumeName" $ }}
+{{- with $.Values.customVolumes.postgres }}
+spec:
+  accessModes:
+    - {{ default "ReadWriteOnce" .accessModes }}
+  persistentVolumeReclaimPolicy: {{ default "Retain" .reclaimPolicy | quote }}
+  capacity:
+    storage: {{ default "8Gi" .size | quote }}
+  storageClassName: {{ include "postgres.storageClassName" $ }}
+  hostPath:
+    path: /var/lib/pgsql/data/userdata
+{{- end }}
+{{- end }}
+{{- end }}

.helm/starter/templates/storage/projects-pv.yaml

@@ -0,0 +1,32 @@
+{{- if and ($.Values.AWX.spec.projects_persistence) (eq (($.Values.AWX.spec).projects_existing_claim | default "") "") -}}
+{{- if (($.Values.customVolumes).projects).enabled -}}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ include "projects.persistentVolumeName" $ }}
+{{- with $.Values.customVolumes.projects }}
+spec:
+  accessModes:
+    - {{ default "ReadWriteOnce" (default $.Values.AWX.spec.projects_storage_access_mode .accessModes) }}
+  persistentVolumeReclaimPolicy: {{ default "Retain" .reclaimPolicy | quote }}
+  capacity:
+    storage: {{ default "8Gi" (default $.Values.AWX.spec.projects_storage_size .size) | quote }}
+  storageClassName: {{ include "projects.storageClassName" $ }}
+  hostPath:
+    path: {{ required "customVolumes.projects.hostPath is required!" .hostPath | quote }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "projects.persistentVolumeClaim" $ }} 
+spec:
+  accessModes:
+    - {{ default "ReadWriteOnce" (default $.Values.AWX.spec.projects_storage_access_mode .accessModes) }}
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ default "8Gi" (default $.Values.AWX.spec.projects_storage_size .size) | quote }}
+  storageClassName: {{ include "projects.storageClassName" $ }}
+{{- end }}  
+{{- end }}
+{{- end }}

CONTRIBUTING.md

@@ -6,13 +6,15 @@ Have questions about this document or anything not covered here? Please file a n
 
 ## Table of contents
 
-* [Things to know prior to submitting code](#things-to-know-prior-to-submitting-code)
-* [Submmiting your Work](#submitting-your-work)
-* [Testing](#testing)
-    * [Testing in Docker](#testing-in-docker)
-    * [Testing in Minikube](#testing-in-minikube)
-* [Generating a bundle](#generating-a-bundle)
-* [Reporting Issues](#reporting-issues)
+- [AWX-Operator Contributing Guidelines](#awx-operator-contributing-guidelines)
+  - [Table of contents](#table-of-contents)
+  - [Things to know prior to submitting code](#things-to-know-prior-to-submitting-code)
+  - [Submmiting your work](#submmiting-your-work)
+  - [Testing](#testing)
+      - [Testing in Kind](#testing-in-kind)
+      - [Testing in Minikube](#testing-in-minikube)
+  - [Generating a bundle](#generating-a-bundle)
+  - [Reporting Issues](#reporting-issues)
 
 
 ## Things to know prior to submitting code
@@ -25,7 +27,7 @@ Have questions about this document or anything not covered here? Please file a n
 
 
 ## Submmiting your work
-1. From your fork `devel` branch, create a new brach to stage your changes.
+1. From your fork `devel` branch, create a new branch to stage your changes.
 ```sh
 #> git checkout -b <branch-name>
 ```
@@ -44,12 +46,12 @@ Have questions about this document or anything not covered here? Please file a n
 
 ## Testing
 
-This Operator includes a [Molecule](https://molecule.readthedocs.io/en/stable/)-based test environment, which can be executed standalone in Docker (e.g. in CI or in a single Docker container anywhere), or inside any kind of Kubernetes cluster (e.g. Minikube).
+This Operator includes a [Molecule](https://ansible.readthedocs.io/projects/molecule/)-based test environment, which can be executed standalone in Docker (e.g. in CI or in a single Docker container anywhere), or inside any kind of Kubernetes cluster (e.g. Minikube).
 
 You need to make sure you have Molecule installed before running the following commands. You can install Molecule with:
 
 ```sh
-#> pip install 'molecule[docker]'
+#> python -m pip install molecule-plugins[docker]
 ```
 
 Running `molecule test` sets up a clean environment, builds the operator, runs all configured tests on an example operator instance, then tears down the environment (at least in the case of Docker).

Dockerfile

@@ -1,8 +1,8 @@
-FROM quay.io/operator-framework/ansible-operator:v1.28.1
+FROM quay.io/operator-framework/ansible-operator:v1.34.0
 
-USER 0
-
-RUN dnf install -y openssl
+USER root
+RUN dnf update --security --bugfix -y && \
+    dnf install -y openssl
 
 USER 1001
 

Makefile

@@ -4,6 +4,7 @@
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= $(shell git describe --tags)
+PREV_VERSION ?= $(shell git describe --abbrev=0 --tags $(shell git rev-list --tags --skip=1 --max-count=1))
 
 CONTAINER_CMD ?= docker
 
@@ -88,6 +89,10 @@ all: docker-build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+.PHONY: print-%
+print-%: ## Print any variable from the Makefile. Use as `make print-VARIABLE`
+	@echo $($*)
+
 ##@ Build
 
 .PHONY: run
@@ -102,6 +107,21 @@ docker-build: ## Build docker image with the manager.
 docker-push: ## Push docker image with the manager.
 	${CONTAINER_CMD} push ${IMG}
 
+# PLATFORMS defines the target platforms for  the manager image be build to provide support to multiple
+# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/
+# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMG=<myregistry/image:<tag>> than the export will fail)
+# To properly provided solutions that supports more than one platform you should use this option.
+PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
+.PHONY: docker-buildx
+docker-buildx: ## Build and push docker image for the manager for cross-platform support
+	- docker buildx create --name project-v3-builder
+	docker buildx use project-v3-builder
+	- docker buildx build --push $(BUILD_ARGS) --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile .
+	- docker buildx rm project-v3-builder
+
+
 ##@ Deployment
 
 .PHONY: install
@@ -149,6 +169,22 @@ KUSTOMIZE = $(shell which kustomize)
 endif
 endif
 
+.PHONY: operator-sdk
+OPERATOR_SDK = $(shell pwd)/bin/operator-sdk
+operator-sdk: ## Download operator-sdk locally if necessary, preferring the $(pwd)/bin path over global if both exist.
+ifeq (,$(wildcard $(OPERATOR_SDK)))
+ifeq (,$(shell which operator-sdk 2>/dev/null))
+	@{ \
+	set -e ;\
+	mkdir -p $(dir $(OPERATOR_SDK)) ;\
+	curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/v1.33.0/operator-sdk_$(OS)_$(ARCHA) ;\
+	chmod +x $(OPERATOR_SDK) ;\
+	}
+else
+OPERATOR_SDK = $(shell which operator-sdk)
+endif
+endif
+
 .PHONY: ansible-operator
 ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator
 ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist.
@@ -157,7 +193,7 @@ ifeq (,$(shell which ansible-operator 2>/dev/null))
 	@{ \
 	set -e ;\
 	mkdir -p $(dir $(ANSIBLE_OPERATOR)) ;\
-	curl -sSLo $(ANSIBLE_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/v1.26.0/ansible-operator_$(OS)_$(ARCHA) ;\
+	curl -sSLo $(ANSIBLE_OPERATOR) https://github.com/operator-framework/ansible-operator-plugins/releases/download/v1.34.0/ansible-operator_$(OS)_$(ARCHA) ;\
 	chmod +x $(ANSIBLE_OPERATOR) ;\
 	}
 else
@@ -166,11 +202,11 @@ endif
 endif
 
 .PHONY: bundle
-bundle: kustomize ## Generate bundle manifests and metadata, then validate generated files.
-	operator-sdk generate kustomize manifests -q
+bundle: kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files.
+	$(OPERATOR_SDK) generate kustomize manifests -q
 	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
-	$(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)
-	operator-sdk bundle validate ./bundle
+	$(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)
+	$(OPERATOR_SDK) bundle validate ./bundle
 
 .PHONY: bundle-build
 bundle-build: ## Build the bundle image.

PROJECT

@@ -1,3 +1,7 @@
+# Code generated by tool. DO NOT EDIT.
+# This file is used to track the info used to scaffold your project
+# and allow the plugins properly work.
+# More info: https://book.kubebuilder.io/reference/project-config.html
 domain: ansible.com
 layout:
 - ansible.sdk.operatorframework.io/v1
@@ -27,4 +31,11 @@ resources:
   group: awx
   kind: AWXRestore
   version: v1beta1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: ansible.com
+  group: awx
+  kind: AWXMeshIngress
+  version: v1alpha1
 version: "3"

README.md

@@ -2,7 +2,7 @@
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Build Status](https://github.com/ansible/awx-operator/workflows/CI/badge.svg?event=push)](https://github.com/ansible/awx-operator/actions)
-[![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html) 
+[![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html)
 [![AWX Mailing List](https://img.shields.io/badge/mailing%20list-AWX-orange.svg)](https://groups.google.com/g/awx-project)
 [![IRC Chat - #ansible-awx](https://img.shields.io/badge/IRC-%23ansible--awx-blueviolet.svg)](https://libera.chat)
 
@@ -12,74 +12,11 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w
 <!-- gh-md-toc --insert README.md -->
 <!--ts-->
 
-**Note**: We are in the process of moving this readme into official docs in the /docs folder. Please go there to find additional sections during this interim move phase.
+# AWX Operator Documentation
 
+The AWX Operator documentation is now available at https://awx-operator.readthedocs.io/
 
-Table of Contents
-=================
-
-- [AWX Operator](#awx-operator)
-- [Table of Contents](#table-of-contents)
-  - [Install and Configuration](#install-and-configuration)
-  - [Contributing](#contributing)
-  - [Release Process](#release-process)
-  - [Author](#author)
-  - [Code of Conduct](#code-of-conduct)
-  - [Get Involved](#get-involved)
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
-
-<!--te-->
-
-
-
-## Install and Configuration
-
-All of our usage and configuration docs are nested in the `docs/` directory. Below is a Table of Contents for those.
-
-- Introduction
-  - [Introduction](./docs/introduction/introduction.md)
-- Contributors Guide
-  - [Code of Conduct](./docs/contributors-guide/code-of-conduct.md)
-  - [Get Involved](./docs/contributors-guide/get-involved.md)
-  - [Contributing](./docs/contributors-guide/contributing.md)
-  - [Release Process](./docs/contributors-guide/release-process.md)
-  - [Authors](./docs/contributors-guide/author.md)
-- Installation
-  - [Basic Install](./docs/installation/basic-install.md)
-  - [Creating a Minikube cluster for testing](./docs/installation/creating-a-minikube-cluster-for-testing.md)
-  - [Helm Install](./docs/installation/helm-install-on-existing-cluster.md)
-- [Migration](./docs/migration/migration.md)
-- [Uninstall](./docs/uninstall/uninstall.md)
-- User Guide
-  - [Admin User Configuration](./docs/user-guide/admin-user-account-configuration.md)
-  - [Database Configuration](./docs/user-guide/database-configuration.md)
-  - [Network and TLS Configuration](./docs/user-guide/network-and-tls-configuration.md)
-  - Advanced Configuration
-    - [No Log](./docs/user-guide/advanced-configuration/no-log.md)
-    - [Deploy a Specific Version of AWX](./docs/user-guide/advanced-configuration/deploying-a-specific-version-of-awx.md)
-    - [Resource Requirements](./docs/user-guide/advanced-configuration/containers-resource-requirements.md)
-    - [Extra Settings](./docs/user-guide/advanced-configuration/extra-settings.md)
-    - [Environment Variables](./docs/user-guide/advanced-configuration/exporting-environment-variables-to-containers.md)
-    - [Custom Labels](./docs/user-guide/advanced-configuration/labeling-operator-managed-objects.md)
-    - [Custom Volumes](./docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md)
-    - [Scaling Deployments](./docs/user-guide/advanced-configuration/scaling-the-web-and-task-pods-independently.md)
-    - [Auto Update Upon Operator Upgrade](./docs/user-guide/advanced-configuration/auto-upgrade.md)
-    - [Termination Grace Period](./docs/user-guide/advanced-configuration/pods-termination-grace-period.md)
-    - [Node Selector for Deployments](./docs/user-guide/advanced-configuration/assigning-awx-pods-to-specific-nodes.md)
-    - [Default EE from Private Registries](./docs/user-guide/advanced-configuration/default-execution-environments-from-private-registries.md)
-    - [CSRF Cookie Secure](./docs/user-guide/advanced-configuration/csrf-cookie-secure-setting.md)
-    - [Disable IPv6](./docs/user-guide/advanced-configuration/disable-ipv6.md)
-    - [LDAP](./docs/user-guide/advanced-configuration/enabling-ldap-integration-at-awx-bootstrap.md)
-    - [Priority Clases](./docs/user-guide/advanced-configuration/priority-classes.md)
-    - [Priveleged Tasks](./docs/user-guide/advanced-configuration/privileged-tasks.md)
-    - [Redis Container Capabilities](./docs/user-guide/advanced-configuration/redis-container-capabilities.md)
-    - [Trusting a Custom Certificate Authority](./docs/user-guide/advanced-configuration/trusting-a-custom-certificate-authority.md)
-    - [Service Account](./docs/user-guide/advanced-configuration/service-account.md)
-    - [Persisting the Projects Directory](./docs/user-guide/advanced-configuration/persisting-projects-directory.md)
-- Troubleshooting
-  - [General Debugging](./docs/troubleshooting/debugging.md)
-
+For docs changes, create PRs on the appropriate files in the /docs folder.
 
 ## Contributing
 
@@ -110,4 +47,3 @@ We welcome your feedback and ideas. The AWX operator uses the same mailing list
 
 - Join the [Ansible AWX channel on Matrix](https://matrix.to/#/#awx:ansible.com)
 - Join the [Ansible Community Forum](https://forum.ansible.com)
-- Join the [mailing list](https://groups.google.com/forum/#!forum/awx-project)

awxmeshingress-demo.yml

@@ -0,0 +1,7 @@
+---
+apiVersion: awx.ansible.com/v1alpha1
+kind: AWXMeshIngress
+metadata:
+  name: awx-mesh-ingress-demo
+spec:
+  deployment_name: awx-demo

config/crd/bases/awx.ansible.com_awxbackups.yaml

@@ -92,7 +92,8 @@ spec:
                 type: string
               precreate_partition_hours:
                 description: Number of hours worth of events table partitions to precreate before backup to avoid pg_dump locks.
-                type: string
+                type: integer
+                format: int32
               image_pull_policy:
                 description: The image pull policy
                 type: string

config/crd/bases/awx.ansible.com_awxmeshingresses.yaml

@@ -0,0 +1,86 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxmeshingresses.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXMeshIngress
+    listKind: AWXMeshIngressList
+    plural: awxmeshingresses
+    singular: awxmeshingress
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: AWXMeshIngress is the Schema for the awxmeshingresses API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec defines the desired state of AWXMeshIngress
+            type: object
+            x-kubernetes-preserve-unknown-fields: true
+            required:
+            - deployment_name
+            properties:
+              deployment_name:
+                description: Name of the AWX deployment to create the Mesh Ingress for.
+                type: string
+              image_pull_secrets:
+                description: Image pull secrets for Mesh Ingress containers.
+                type: array
+                items:
+                  type: string
+              external_hostname:
+                description: External hostname to use for the Mesh Ingress.
+                type: string
+              external_ipaddress:
+                description: External IP address to use for the Mesh Ingress.
+                type: string
+              ingress_type:
+                description: The ingress type to use to reach the deployed instance
+                type: string
+                enum:
+                - none
+                - Ingress
+                - ingress
+                - IngressRouteTCP
+                - ingressroutetcp
+                - Route
+                - route
+              ingress_api_version:
+                description: The Ingress API version to use
+                type: string
+              ingress_annotations:
+                description: Annotations to add to the Ingress Controller
+                type: string
+              ingress_class_name:
+                description: The name of ingress class to use instead of the cluster default.
+                type: string
+              ingress_controller:
+                description: Special configuration for specific Ingress Controllers
+                type: string
+          status:
+            description: Status defines the observed state of AWXMeshIngress
+            type: object
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/bases/awx.ansible.com_awxrestores.yaml

@@ -94,6 +94,11 @@ spec:
               postgres_image_version:
                 description: PostgreSQL container image version to use
                 type: string
+              spec_overrides:
+                description: Overrides for the AWX spec
+                # type: string
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
               image_pull_policy:
                 description: The image pull policy
                 type: string
@@ -121,6 +126,10 @@ spec:
                 description: Maintain some of the recommended `app.kubernetes.io/*` labels on the resource (self)
                 type: boolean
                 default: true
+              force_drop_db:
+                description: Force drop the database before restoring. USE WITH CAUTION!
+                type: boolean
+                default: false
           status:
             type: object
             x-kubernetes-preserve-unknown-fields: true

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -54,8 +54,8 @@ spec:
                 description: Username to use for the admin account
                 type: string
                 default: admin
-              hostname:
-                description: The hostname of the instance
+              hostname:  # deprecated
+                description: (Deprecated) The hostname of the instance
                 type: string
               admin_email:
                 description: The admin user email
@@ -123,15 +123,30 @@ spec:
               ingress_annotations:
                 description: Annotations to add to the Ingress Controller
                 type: string
-              ingress_tls_secret:
-                description: Secret where the Ingress TLS secret can be found
+              ingress_tls_secret:  # deprecated
+                description: (Deprecated) Secret where the Ingress TLS secret can be found
                 type: string
               ingress_class_name:
                 description: The name of ingress class to use instead of the cluster default.
                 type: string
+              ingress_hosts:
+                description: Ingress hostnames of the instance
+                type: array
+                items:
+                  type: object
+                  properties:
+                    hostname:
+                      description: Hostname of the instance
+                      type: string
+                    tls_secret:
+                      description: Secret where the Ingress TLS secret can be found
+                      type: string
               ingress_controller:
                 description: Special configuration for specific Ingress Controllers
                 type: string
+              api_urlpattern_prefix:
+                description: An optional configuration to add a prefix in the API URL path
+                type: string
               loadbalancer_protocol:
                 description: Protocol to use for the loadbalancer
                 type: string
@@ -147,6 +162,10 @@ spec:
                 description: Assign LoadBalancer IP address
                 type: string
                 default: ''
+              loadbalancer_class:
+                description: Class of LoadBalancer to use
+                type: string
+                default: ''
               route_host:
                 description: The DNS to use to points to the instance
                 type: string
@@ -1444,7 +1463,7 @@ spec:
                     type: object
                 type: object
               postgres_init_container_resource_requirements:
-                description: Resource requirements for the postgres init container
+                description: (Deprecated, use postgres_resource_requirements parameter) Resource requirements for the postgres init container
                 properties:
                   requests:
                     properties:
@@ -1555,10 +1574,98 @@ spec:
                 description: Number of web instance replicas
                 type: integer
                 format: int32
+              web_manage_replicas:
+                description: Enables operator control of replicas count for the web deployment when set to 'true'
+                type: boolean
+                default: true
               task_replicas:
                 description: Number of task instance replicas
                 type: integer
                 format: int32
+              task_manage_replicas:
+                description: Enables operator control of replicas count for the task deployment when set to 'true'
+                type: boolean
+                default: true
+              web_liveness_initial_delay:
+                description: Initial delay before starting liveness checks on web pod
+                type: integer
+                default: 5
+                format: int32
+              task_liveness_initial_delay:
+                description: Initial delay before starting liveness checks on task pod
+                type: integer
+                default: 5
+                format: int32
+              web_liveness_period:
+                description: Time period in seconds between each liveness check for the web pod
+                type: integer
+                default: 0
+                format: int32
+              task_liveness_period:
+                description: Time period in seconds between each liveness check for the task pod
+                type: integer
+                default: 0
+                format: int32
+              web_liveness_failure_threshold:
+                description: Number of consecutive failure events to identify failure of web pod
+                type: integer
+                default: 3
+                format: int32
+              task_liveness_failure_threshold:
+                description: Number of consecutive failure events to identify failure of task pod
+                type: integer
+                default: 3
+                format: int32
+              web_liveness_timeout:
+                description: Number of seconds to wait for a probe response from web pod
+                type: integer
+                default: 1
+                format: int32
+              task_liveness_timeout:
+                description: Number of seconds to wait for a probe response from task pod
+                type: integer
+                default: 1
+                format: int32
+              web_readiness_initial_delay:
+                description: Initial delay before starting readiness checks on web pod
+                type: integer
+                default: 20
+                format: int32
+              task_readiness_initial_delay:
+                description: Initial delay before starting readiness checks on task pod
+                type: integer
+                default: 20
+                format: int32
+              web_readiness_period:
+                description: Time period in seconds between each readiness check for the web pod
+                type: integer
+                default: 0
+                format: int32
+              task_readiness_period:
+                description: Time period in seconds between each readiness check for the task pod
+                type: integer
+                default: 0
+                format: int32
+              web_readiness_failure_threshold:
+                description: Number of consecutive failure events to identify failure of web pod
+                type: integer
+                default: 3
+                format: int32
+              task_readiness_failure_threshold:
+                description: Number of consecutive failure events to identify failure of task pod
+                type: integer
+                default: 3
+                format: int32
+              web_readiness_timeout:
+                description: Number of seconds to wait for a probe response from web pod
+                type: integer
+                default: 1
+                format: int32
+              task_readiness_timeout:
+                description: Number of seconds to wait for a probe response from task pod
+                type: integer
+                default: 1
+                format: int32
               garbage_collect_secrets:
                 description: Whether or not to remove secrets upon instance removal
                 default: false
@@ -1608,6 +1715,9 @@ spec:
               web_extra_volume_mounts:
                 description: Specify volume mounts to be added to the Web container
                 type: string
+              postgres_extra_volume_mounts:
+                description: Specify volume mounts to be added to Postgres container
+                type: string
               uwsgi_processes:
                 description: Set the number of uwsgi processes to run in a web container
                 type: integer
@@ -1708,13 +1818,19 @@ spec:
               postgres_priority_class:
                 description: Assign a preexisting priority class to the postgres pod
                 type: string
-              postgres_data_path:
-                description: Path where the PostgreSQL data are located
-                type: string
               postgres_extra_args:
                 type: array
                 items:
                   type: string
+              postgres_data_volume_init:
+                description: Sets permissions on the /var/lib/pgdata/data for postgres container using an init container (not Openshift)
+                type: boolean
+              postgres_init_container_commands:
+                description: Customize the postgres init container commands (Non Openshift)
+                type: string
+              postgres_extra_volumes:
+                description: Specify extra volumes to add to the application pod
+                type: string
               postgres_keepalives:
                 description: Controls whether client-side TCP keepalives are used for Postgres connections.
                 default: true
@@ -1779,11 +1895,15 @@ spec:
               session_cookie_secure:
                 description: Set session cookie secure mode for web
                 type: string
+              postgres_security_context_settings:
+                description: Key/values that will be set under the pod-level securityContext field
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
               receptor_log_level:
                 description: Set log level of receptor service
                 type: string
               extra_settings:
-                description: Extra settings to specify for the API
+                description: Extra settings to specify for AWX
                 items:
                   properties:
                     setting:
@@ -1792,6 +1912,28 @@ spec:
                       x-kubernetes-preserve-unknown-fields: true
                   type: object
                 type: array
+              extra_settings_files:
+                description: Extra ConfigMaps or Secrets of settings files to specify for AWX
+                properties:
+                  configmaps:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                  secrets:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                type: object
               no_log:
                 description: Configure no_log for no_log tasks
                 type: boolean
@@ -1817,6 +1959,50 @@ spec:
                 description: Disable web container's nginx ipv6 listener
                 type: boolean
                 default: false
+              metrics_utility_enabled:
+                description: Enable metrics utility
+                type: boolean
+                default: false
+              metrics_utility_image:
+                description: Metrics-Utility Image
+                type: string
+              metrics_utility_image_version:
+                description: Metrics-Utility Image Version
+                type: string
+              metrics_utility_image_pull_policy:
+                description: Metrics-Utility Image PullPolicy
+                type: string
+              metrics_utility_configmap:
+                description: Metrics-Utility ConfigMap
+                type: string
+              metrics_utility_secret:
+                description: Metrics-Utility Secret
+                type: string
+              metrics_utility_cronjob_gather_schedule:
+                description: Metrics-Utility Gather Data CronJob Schedule
+                type: string
+                default: '@hourly'
+              metrics_utility_cronjob_report_schedule:
+                description: Metrics-Utility Report CronJob Schedule
+                type: string
+                default: '@monthly'
+              metrics_utility_ship_target:
+                description: Metrics-Utility Ship Target
+                type: string
+              metrics_utility_pvc_claim:
+                description: Metrics-Utility PVC Claim
+                type: string
+              metrics_utility_pvc_claim_size:
+                description: Metrics-Utility PVC Claim Size
+                type: string
+                default: 5Gi
+              metrics_utility_pvc_claim_storage_class:
+                description: Metrics-Utility PVC Claim Storage Class
+                type: string
+              metrics_utility_console_enabled:
+                description: Enable metrics utility shipping to Red Hat Hybrid Cloud Console
+                type: boolean
+                default: false
             type: object
           status:
             properties:

config/crd/kustomization.yaml

@@ -5,4 +5,5 @@ resources:
 - bases/awx.ansible.com_awxs.yaml
 - bases/awx.ansible.com_awxbackups.yaml
 - bases/awx.ansible.com_awxrestores.yaml
+- bases/awx.ansible.com_awxmeshingresses.yaml
 #+kubebuilder:scaffold:crdkustomizeresource

config/default/manager_auth_proxy_patch.yaml

@@ -15,7 +15,7 @@ spec:
           capabilities:
             drop:
             - "ALL"
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1
+        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8080/"

config/manager/kustomization.yaml

@@ -5,9 +5,9 @@ generatorOptions:
   disableNameSuffixHash: true
 
 configMapGenerator:
-- name: awx-manager-config
-  files:
+- files:
   - controller_manager_config.yaml
+  name: awx-manager-config
 
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

config/manager/manager.yaml

@@ -39,6 +39,7 @@ spec:
         - --leader-elect
         - --leader-election-id=awx-operator
         image: controller:latest
+        imagePullPolicy: IfNotPresent
         name: awx-manager
         env:
         - name: ANSIBLE_GATHERING
@@ -72,8 +73,8 @@ spec:
             memory: "32Mi"
             cpu: "50m"
           limits:
-            memory: "4096Mi"
-            cpu: "2000m"
+            memory: "960Mi"
+            cpu: "1500m"
       serviceAccountName: controller-manager
       imagePullSecrets:
       - name: redhat-operators-pull-secret

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -10,12 +10,65 @@ metadata:
     description: AWX provides a web-based user interface, REST API, and task engine
       built on top of Ansible.
     repository: https://github.com/ansible/awx-operator
+    support: forum.ansible.com
   name: awx-operator.v0.0.0
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
   customresourcedefinitions:
     owned:
+    - description: Deploy a instance of AWX Mesh ingress to allow inbound connection
+        to the AWX Receptor Mesh.
+      displayName: AWX Mesh Ingress
+      kind: AWXMeshIngress
+      name: awxmeshingresses.awx.ansible.com
+      specDescriptors:
+      - displayName: Deployment Name
+        path: deployment_name
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: External Hostname
+        path: external_hostname
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: External IP Address
+        path: external_ipaddress
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Ingress Type
+        path: ingress_type
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:none
+        - urn:alm:descriptor:com.tectonic.ui:select:Ingress
+        - urn:alm:descriptor:com.tectonic.ui:select:IngressRouteTCP
+        - urn:alm:descriptor:com.tectonic.ui:select:Route
+      - displayName: Ingress API Version
+        path: ingress_api_version
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Ingress Annotations
+        path: ingress_annotations
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Ingress Class Name
+        path: ingress_class_name
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Ingress Controller
+        path: ingress_controller
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Optional API URLPATTERN Prefix
+        path: api_urlpattern_prefix
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Image Pull Secrets
+        path: image_pull_secrets
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:Secret
+      version: v1alpha1
     - description: Back up a deployment of the awx, including jobs, inventories, and
         credentials
       displayName: AWX Backup
@@ -48,12 +101,12 @@ spec:
       - displayName: Backup PVC Storage Class
         path: backup_storage_class
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Precreate Partition Hours
         path: precreate_partition_hours
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:number
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Database Backup Label Selector
@@ -61,6 +114,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: PostgreSQL Security Context Settings
+        path: postgres_security_context_settings
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: PostgreSQL Image
         path: postgres_image
         x-descriptors:
@@ -160,6 +218,10 @@ spec:
         path: postgres_image_version
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: AWX Spec Overrides
+        path: spec_overrides
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Image Pull Policy
         path: image_pull_policy
         x-descriptors:
@@ -189,6 +251,11 @@ spec:
         path: db_management_pod_node_selector
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Force drop database before restore
+        path: force_drop_db
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
       statusDescriptors:
       - description: The state of the restore
         displayName: Restore Status
@@ -202,7 +269,7 @@ spec:
       kind: AWX
       name: awxs.awx.ansible.com
       specDescriptors:
-      - displayName: Hostname
+      - displayName: Hostname (Deprecated)
         path: hostname
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
@@ -292,12 +359,17 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:Ingress
-      - displayName: Ingress TLS Secret
+      - displayName: Ingress TLS Secret (Deprecated)
         path: ingress_tls_secret
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:io.kubernetes:Secret
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:Ingress
+      - displayName: Ingress Hosts
+        path: ingress_hosts
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
       - displayName: Ingress Controller
         path: ingress_controller
         x-descriptors:
@@ -329,6 +401,12 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:string
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer
+      - displayName: LoadBalancer Class
+        path: loadbalancer_class
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:string
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer
       - displayName: Route API Version
         path: route_api_version
         x-descriptors:
@@ -387,12 +465,21 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
-      - description: The PostgreSQL init container is not used when an external DB
-          is configured
+      - description: Sets permissions on the /var/lib/pgsql/data for postgres container using an init container (not Openshift)
+        displayName: PostgreSQL initialize data volume
+        path: postgres_data_volume_init
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: Customize the postgres init container commands (Non Openshift)
+        displayName: PostgreSQL Init Container Commands
+        path: postgres_init_container_commands
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: (Deprecated, use postgres_resource_requirements parameter instead)
         displayName: PostgreSQL Init Container Resource Requirements
         path: postgres_init_container_resource_requirements
         x-descriptors:
-        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
         - urn:alm:descriptor:com.tectonic.ui:resourceRequirements
       - displayName: Redis Container Resource Requirements
         path: redis_resource_requirements
@@ -568,13 +655,20 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:StorageClass
         - urn:alm:descriptor:com.tectonic.ui:advanced
-      - displayName: Postgres Datapath
-        path: postgres_data_path
+      - displayName: Postgres Extra Arguments
+        path: postgres_extra_args
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
-      - displayName: Postgres Extra Arguments
-        path: postgres_extra_args
+      - description: Specify extra volumes to add to the postgres pod
+        displayName: Postgres Extra Volumes
+        path: postgres_extra_volumes
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      - description: Specify volume mounts to be added to Postgres container
+        displayName: Postgres Extra Volume Mounts
+        path: postgres_extra_volume_mounts
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
@@ -643,7 +737,7 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:projects_use_existing_claim:_No_
-        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:io.kubernetes:StorageClass
       - description: Projects Storage Size
         displayName: Projects Storage Size
         path: projects_storage_size
@@ -867,11 +961,16 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
-      - displayName: API Extra Settings
+      - displayName: Extra Settings
         path: extra_settings
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Extra Settings Files
+        path: extra_settings_files
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: No Log Configuration
         path: no_log
         x-descriptors:
@@ -943,6 +1042,83 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - displayName: Metrics-Utility Enabled
+        path: metrics_utility_enabled
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - displayName: Metrics-Utility Image
+        path: metrics_utility_image
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Image Version
+        path: metrics_utility_image_version
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Image PullPolicy
+        path: metrics_utility_image_pull_policy
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:imagePullPolicy
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility ConfigMap
+        path: metrics_utility_configmap
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:ConfigMap
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Secret
+        path: metrics_utility_secret
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:Secret
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Gather Data CronJob Schedule
+        path: metrics_utility_cronjob_gather_schedule
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Report CronJob Schedule
+        path: metrics_utility_cronjob_report_schedule
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Ship Target
+        path: metrics_utility_ship_target
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility PVC Claim
+        path: metrics_utility_pvc_claim
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility PVC Claim Size
+        path: metrics_utility_pvc_claim_size
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility PVC Claim Storage Class
+        path: metrics_utility_pvc_claim_storage_class
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:io.kubernetes:StorageClass
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
+      - displayName: Metrics-Utility Enabled Shipping to Red Hat Hybrid Cloud Console
+        path: metrics_utility_console_enabled
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true
       statusDescriptors:
       - description: Route to access the instance deployed
         displayName: URL

config/rbac/awxmeshingress_editor_role.yaml

@@ -0,0 +1,31 @@
+# permissions for end users to edit awxmeshingresses.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: clusterrole
+    app.kubernetes.io/instance: awxmeshingress-editor-role
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: awx-operator
+    app.kubernetes.io/part-of: awx-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: awxmeshingress-editor-role
+rules:
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses/status
+  verbs:
+  - get

config/rbac/awxmeshingress_viewer_role.yaml

@@ -0,0 +1,27 @@
+# permissions for end users to view awxmeshingresses.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: clusterrole
+    app.kubernetes.io/instance: awxmeshingress-viewer-role
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: awx-operator
+    app.kubernetes.io/part-of: awx-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: awxmeshingress-viewer-role
+rules:
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses/status
+  verbs:
+  - get

config/rbac/role.yaml

@@ -78,6 +78,18 @@ rules:
       - patch
       - update
       - watch
+  - apiGroups:
+      - batch
+    resources:
+      - cronjobs
+      - jobs
+    verbs:
+      - get
+      - list
+      - create
+      - patch
+      - update
+      - watch
   - apiGroups:
       - monitoring.coreos.com
     resources:
@@ -124,3 +136,16 @@ rules:
       - awxrestores
     verbs:
       - '*'
+  - apiGroups:
+      - traefik.containo.us
+      - traefik.io
+    resources:
+      - ingressroutetcps
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - patch
+      - update
+      - watch

config/samples/awx_v1alpha1_awxmeshingress.yaml

@@ -0,0 +1,8 @@
+# Placeholder to pass CI and allow bundle generation
+---
+apiVersion: awx.ansible.com/v1alpha1
+kind: AWXMeshIngress
+metadata:
+  name: example-awx-mesh-ingress
+spec:
+  deployment_name: example-awx

config/samples/awx_v1beta1_awx_resource_limits.yaml

@@ -0,0 +1,48 @@
+---
+apiVersion: awx.ansible.com/v1beta1
+kind: AWX
+metadata:
+  name: awx-with-limits
+spec:
+  task_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  web_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 1000m
+      memory: 4Gi
+  ee_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 64Mi
+    limits:
+      cpu: 1000m
+      memory: 4Gi
+  redis_resource_requirements:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 1000m
+      memory: 4Gi
+  rsyslog_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 1000m
+      memory: 2Gi
+  init_container_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 1000m
+      memory: 2Gi

config/samples/kustomization.yaml

@@ -3,4 +3,5 @@ resources:
 - awx_v1beta1_awx.yaml
 - awx_v1beta1_awxbackup.yaml
 - awx_v1beta1_awxrestore.yaml
+- awx_v1alpha1_awxmeshingress.yaml
 #+kubebuilder:scaffold:manifestskustomizesamples

docs/README.md

@@ -2,9 +2,15 @@
 
 To build the AWX Operator docs locally:
 
-1. Clone the AWX operator repository. 
-2. From the root directory:
-    a.  pip install --user -r docs/requirements.txt
-    b.  mkdocs build
+1. Clone the AWX operator repository.
+1. Preferrably, create a virtual environment for installing the dependencies.  
+   a. `python3 -m venv venv`  
+   b. `source venv/bin/activate`
+1. From the root directory:  
+   a. `pip install -r docs/requirements.txt`  
+   b. `mkdocs build`
+1. View the docs in your browser:  
+   a. `mkdocs serve`  
+   b. Open your browser and navigate to `http://127.0.0.1:8000/`
 
-This will create a new directory called `site/` in the root of your clone containing the index.html and static files. To view the docs in your browser, navigate there in your file explorer and double-click on the `index.html` file. This should open the docs site in your browser.
+This will create a new directory called `site/` in the root of your clone containing the index.html and static files.

docs/contributors-guide/release-process.md

@@ -9,7 +9,7 @@ After the draft release is created, publish it and the [Promote AWX Operator ima
 - Publish image to Quay
 - Release Helm chart
 
-After the GHA is complete, the final step is to run the [publish-to-operator-hub.sh](./hack/publish-to-operator-hub.sh) script, which will create a PR in the following repos to add the new awx-operator bundle version to OperatorHub:
+After the GHA is complete, the final step is to run the [publish-to-operator-hub.sh](https://github.com/ansible/awx-operator/blob/devel/hack/publish-to-operator-hub.sh) script, which will create a PR in the following repos to add the new awx-operator bundle version to OperatorHub:
 * https://github.com/k8s-operatorhub/community-operators (community operator index)
 * https://github.com/redhat-openshift-ecosystem/community-operators-prod (operator index shipped with Openshift)
 

docs/installation/basic-install.md

@@ -1,7 +1,29 @@
 ### Basic Install
 
+After cloning this repository, you must choose the tag to run:
+```sh
+git clone git@github.com:ansible/awx-operator.git
+cd awx-operator
+git tag
+git checkout tags/<tag>
+
+# For instance:
+git checkout tags/2.7.2
+```
+
+If you work from a fork and made modifications since the tag was issued, you must provide the VERSION number to deploy. Otherwise the operator will get stuck in "ImagePullBackOff" state:
+
+```sh
+export VERSION=<tag>
+
+# For instance:
+export VERSION=2.7.2
+```
+
 Once you have a running Kubernetes cluster, you can deploy AWX Operator into your cluster using [Kustomize](https://kubectl.docs.kubernetes.io/guides/introduction/kustomize/). Since kubectl version 1.14 kustomize functionality is built-in (otherwise, follow the instructions here to install the latest version of Kustomize: https://kubectl.docs.kubernetes.io/installation/kustomize/ )
 
+> Some things may need to be configured slightly differently for different Kubernetes flavors for the networking aspects. When installing on Kind, see the [kind install docs](./kind-install.md) for more details.
+
 There is a make target you can run:
 ```
 make deploy
@@ -67,7 +89,7 @@ So we don't have to keep repeating `-n awx`, let's set the current namespace for
 $ kubectl config set-context --current --namespace=awx
 ```
 
-Next, create a file named `awx-demo.yaml` in the same folder with the suggested content below. The `metadata.name` you provide will be the name of the resulting AWX deployment.
+Next, create a file named `awx-demo.yml` in the same folder with the suggested content below. The `metadata.name` you provide will be the name of the resulting AWX deployment.
 
 **Note:** If you deploy more than one AWX instance to the same namespace, be sure to use unique names.
 
@@ -81,7 +103,7 @@ spec:
   service_type: nodeport
 ```
 
-> It may make sense to create and specify your own secret key for your deployment so that if the k8s secret gets deleted, it can be re-created if needed.  If it is not provided, one will be auto-generated, but cannot be recovered if lost. Read more [here](#secret-key-configuration).
+> It may make sense to create and specify your own secret key for your deployment so that if the k8s secret gets deleted, it can be re-created if needed.  If it is not provided, one will be auto-generated, but cannot be recovered if lost. Read more [here](../user-guide/admin-user-account-configuration.md#secret-key-configuration).
 
 If you are on Openshift, you can take advantage of Routes by specifying the following your spec. This will automatically create a Route for you with a custom hostname. This can be found on the Route section of the Openshift Console.
 
@@ -104,7 +126,7 @@ Make sure to add this new file to the list of "resources" in your `kustomization
 resources:
   - github.com/ansible/awx-operator/config/default?ref=<tag>
   # Add this extra line:
-  - awx-demo.yaml
+  - awx-demo.yml
 ...
 ```
 

docs/installation/index.md

@@ -1,2 +1,2 @@
 
-This Kubernetes Operator is meant to be deployed in your Kubernetes cluster(s) and can manage one or more AWX instances in any namespace.
+This Kubernetes Operator is meant to be deployed in your Kubernetes cluster(s) and can be used to install and manage the lifecycle of an AWX instance in the same namespace.

docs/installation/kind-install.md

@@ -0,0 +1,125 @@
+# AWX Operator on Kind
+
+## Kind Install
+
+Install Kind by running the following
+
+```
+# For Intel Macs
+[ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-darwin-amd64
+# For M1 / ARM Macs
+[ $(uname -m) = arm64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-darwin-arm64
+chmod +x ./kind
+mv ./kind /some-dir-in-your-PATH/kind
+```
+
+> https://kind.sigs.k8s.io/docs/user/quick-start/
+
+
+### Create the Kind cluster
+
+Create a file called `kind.config`
+
+```yaml
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+nodes:
+- role: control-plane
+  extraPortMappings:
+  - containerPort: 32000
+    hostPort: 32000
+    listenAddress: "0.0.0.0" # Optional, defaults to "0.0.0.0"
+    protocol: tcp # Optional, defaults to tcp
+- role: worker
+```
+
+Then create a cluster using that config
+
+```
+kind create cluster --config=kind.config
+```
+
+Set cluster context for kubectl
+
+```
+kubectl cluster-info --context kind-kind
+```
+
+Install NGINX Ingress Controller
+
+```
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+```
+
+
+## AWX
+
+Set the namespace context
+
+```
+kubectl config set-context --current --namespace=awx
+```
+
+Checkout the tag you want to install from
+
+```
+git checkout 2.7.2
+```
+
+Create a file named `kustomization.yaml` in the root of your local awx-operator clone. Include the following:
+
+```
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # Find the latest tag here: https://github.com/ansible/awx-operator/releases
+  - github.com/ansible/awx-operator/config/default?ref=2.7.2
+
+# Set the image tags to match the git version from above
+images:
+  - name: quay.io/ansible/awx-operator
+    newTag: 2.7.2
+
+# Specify a custom namespace in which to install AWX
+namespace: awx
+```
+
+Run the following to apply the yaml
+
+```
+kubectl apply -k .
+```
+
+
+Create a file called `awx-cr.yaml` with the following contents and any configuration changes you may wish to add.
+
+```
+---
+apiVersion: awx.ansible.com/v1beta1
+kind: AWX
+metadata:
+  name: awx-demo
+spec:
+  service_type: nodeport
+  nodeport_port: 32000
+```
+
+Create your AWX CR
+
+```
+oc create -f awx-cr.yaml
+```
+
+Your AWX instance should now be reacheable at http://localhost:32000/
+
+> If you configured a custom nodeport_port, you can find it by running `kubectl -n awx get svc awx-demo-service`
+
+
+
+## Cleanup
+
+When you are done, you can delete all of this by running
+
+```
+kind delete cluster
+```

docs/migration/migration.md

@@ -41,7 +41,7 @@ stringData:
 type: Opaque
 ```
 
-> For `host`, a URL resolvable by the cluster could look something like `postgresql.<namespace>.svc.cluster.local`, where `<namespace>` is filled in with the namespace of the AWX deployment you are migrating data from.
+> For `host`, a URL resolvable by the cluster could look something like `postgresql.<namespace>.svc.<cluster domain>`, where `<namespace>` is filled in with the namespace of the AWX deployment you are migrating data from, and `<cluster domain>` is filled in with the internal kubernretes cluster domain (In most cases it's `cluster.local`).
 
 If your AWX deployment is already using an external database server or its database is otherwise not managed
 by the AWX deployment, you can instead create the same secret as above but omit the `-old-` from the `name`.

docs/requirements.in

@@ -0,0 +1,3 @@
+# This requirements file is used for AWX Operator latest doc builds.
+
+mkdocs-ansible

docs/requirements.txt

@@ -1,12 +1,193 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile --allow-unsafe --output-file=docs/requirements.txt --strip-extras docs/requirements.in
+#
+babel==2.14.0
+    # via mkdocs-material
+beautifulsoup4==4.12.3
+    # via
+    #   linkchecker
+    #   mkdocs-htmlproofer-plugin
+    #   readtime
+cairocffi==1.6.1
+    # via cairosvg
 cairosvg==2.7.0
-markdown-exec>=1.6.0
-mkdocs-ansible>=0.1.6
-mkdocs-gen-files>=0.4.0
-mkdocs-material-extensions>=1.1.1
-mkdocs-material>=9.1.18
-mkdocs
-mkdocstrings-python>=1.1.0
-mkdocstrings>=0.22.0
-pillow==9.5.0
+    # via mkdocs-ansible
+certifi==2024.2.2
+    # via requests
+cffi==1.16.0
+    # via cairocffi
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via
+    #   mkdocs
+    #   mkdocstrings
+colorama==0.4.6
+    # via
+    #   griffe
+    #   mkdocs-material
+csscompressor==0.9.5
+    # via mkdocs-minify-plugin
+cssselect==1.2.0
+    # via pyquery
+cssselect2==0.7.0
+    # via cairosvg
+defusedxml==0.7.1
+    # via cairosvg
+dnspython==2.6.1
+    # via linkchecker
+ghp-import==2.1.0
+    # via mkdocs
+griffe==0.40.1
+    # via mkdocstrings-python
+htmlmin2==0.1.13
+    # via mkdocs-minify-plugin
+idna==3.6
+    # via requests
+jinja2==3.1.3
+    # via
+    #   mkdocs
+    #   mkdocs-macros-plugin
+    #   mkdocs-material
+    #   mkdocstrings
+jsmin==3.0.1
+    # via mkdocs-minify-plugin
+linkchecker==10.4.0
+    # via mkdocs-ansible
+lxml==5.1.0
+    # via
+    #   mkdocs-material
+    #   pyquery
+markdown==3.5.2
+    # via
+    #   markdown-include
+    #   mkdocs
+    #   mkdocs-autorefs
+    #   mkdocs-htmlproofer-plugin
+    #   mkdocs-material
+    #   mkdocstrings
+    #   pymdown-extensions
+markdown-exec==1.8.0
+    # via mkdocs-ansible
+markdown-include==0.8.1
+    # via mkdocs-ansible
+markdown2==2.4.12
+    # via readtime
+markupsafe==2.1.5
+    # via
+    #   jinja2
+    #   mkdocs
+    #   mkdocstrings
+mergedeep==1.3.4
+    # via mkdocs
+mkdocs==1.5.3
+    # via
+    #   mkdocs-ansible
+    #   mkdocs-autorefs
+    #   mkdocs-gen-files
+    #   mkdocs-htmlproofer-plugin
+    #   mkdocs-macros-plugin
+    #   mkdocs-material
+    #   mkdocs-minify-plugin
+    #   mkdocs-monorepo-plugin
+    #   mkdocstrings
+mkdocs-ansible==24.3.1
+    # via -r requirements.in
+mkdocs-autorefs==0.5.0
+    # via mkdocstrings
+mkdocs-gen-files==0.5.0
+    # via mkdocs-ansible
+mkdocs-htmlproofer-plugin==1.0.0
+    # via mkdocs-ansible
+mkdocs-macros-plugin==1.0.5
+    # via mkdocs-ansible
+mkdocs-material==9.2.6
+    # via mkdocs-ansible
+mkdocs-material-extensions==1.3.1
+    # via
+    #   mkdocs-ansible
+    #   mkdocs-material
+mkdocs-minify-plugin==0.8.0
+    # via mkdocs-ansible
+mkdocs-monorepo-plugin==1.1.0
+    # via mkdocs-ansible
+mkdocstrings==0.24.0
+    # via
+    #   mkdocs-ansible
+    #   mkdocstrings-python
+mkdocstrings-python==1.8.0
+    # via mkdocs-ansible
+packaging==23.2
+    # via mkdocs
+paginate==0.5.6
+    # via mkdocs-material
+pathspec==0.12.1
+    # via mkdocs
+pillow==10.0.1
+    # via
+    #   cairosvg
+    #   mkdocs-ansible
 pipdeptree==2.7.1
+    # via mkdocs-ansible
+platformdirs==4.2.0
+    # via
+    #   mkdocs
+    #   mkdocstrings
+pycparser==2.21
+    # via cffi
+pygments==2.17.2
+    # via mkdocs-material
 pymdown-extensions==10.0.1
+    # via
+    #   markdown-exec
+    #   mkdocs-ansible
+    #   mkdocs-material
+    #   mkdocstrings
+pyquery==2.0.0
+    # via readtime
+python-dateutil==2.8.2
+    # via
+    #   ghp-import
+    #   mkdocs-macros-plugin
+python-slugify==8.0.4
+    # via mkdocs-monorepo-plugin
+pyyaml==6.0.1
+    # via
+    #   mkdocs
+    #   mkdocs-macros-plugin
+    #   pymdown-extensions
+    #   pyyaml-env-tag
+pyyaml-env-tag==0.1
+    # via mkdocs
+readtime==3.0.0
+    # via mkdocs-material
+regex==2023.12.25
+    # via mkdocs-material
+requests==2.31.0
+    # via
+    #   linkchecker
+    #   mkdocs-htmlproofer-plugin
+    #   mkdocs-material
+six==1.16.0
+    # via python-dateutil
+soupsieve==2.5
+    # via beautifulsoup4
+termcolor==2.4.0
+    # via mkdocs-macros-plugin
+text-unidecode==1.3
+    # via python-slugify
+tinycss2==1.2.1
+    # via
+    #   cairosvg
+    #   cssselect2
+urllib3==2.2.1
+    # via requests
+watchdog==4.0.0
+    # via mkdocs
+webencodings==0.5.1
+    # via
+    #   cssselect2
+    #   tinycss2

docs/upgrade/upgrading.md

@@ -1,6 +1,11 @@
 ### Upgrading
 
-To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX.  To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `image_version` variable in `roles/installer/defaults/main.yml` for that particular release.
+To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX. To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command
+
+```shell
+AWX_OPERATOR_VERSION=2.8.0
+docker run --entrypoint="" quay.io/ansible/awx-operator:$AWX_OPERATOR_VERSION bash -c "env | grep DEFAULT_AWX_VERSION"
+```
 
 Apply the awx-operator.yml for that release to upgrade the operator, and in turn also upgrade your AWX deployment.
 
@@ -8,26 +13,23 @@ Apply the awx-operator.yml for that release to upgrade the operator, and in turn
 
 The first part of any upgrade should be a backup. Note, there are secrets in the pod which work in conjunction with the database. Having just a database backup without the required secrets will not be sufficient for recovering from an issue when upgrading to a new version. See the [backup role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/backup) for information on how to backup your database and secrets.
 
-In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). *Before Restoring from a backup*, be sure to:
-* delete the old existing AWX CR
-* delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-13-<deployment-name>-postgres-13-0`
+In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). _Before Restoring from a backup_, be sure to:
+
+- delete the old existing AWX CR
+- delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
 
 **Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well.
 
-
 #### PostgreSQL Upgrade Considerations
 
 If there is a PostgreSQL major version upgrade, after the data directory on the PVC is migrated to the new version, the old PVC is kept by default.
-This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically
-after a successful upgrade by setting the following variable on the AWX spec. 
-
+This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically after a successful upgrade by setting the following variable on the AWX spec.
 
 ```yaml
-  spec:
-    postgres_keep_pvc_after_upgrade: False
+spec:
+  postgres_keep_pvc_after_upgrade: False
 ```
 
-
 #### v0.14.0
 
 ##### Cluster-scope to Namespace-scope considerations

docs/user-guide/advanced-configuration/adding-execution-nodes.md

@@ -0,0 +1,4 @@
+### Adding Execution Nodes
+Starting with AWX Operator v0.30.0 and AWX v21.7.0, standalone execution nodes can be added to your deployments.
+See [Managing Capacity With Instances](https://ansible.readthedocs.io/projects/awx/en/latest/administration/instances.html) chapter of the AWX Administration Guide for information about this feature.
+

docs/user-guide/advanced-configuration/assigning-awx-pods-to-specific-nodes.md

@@ -9,25 +9,25 @@ If you want to use affinity rules for your AWX pod you can use the `affinity` op
 If you want to constrain the web and task pods individually, you can do so by specificying the deployment type before the specific setting. For
 example, specifying `task_tolerations` will allow the AWX task pod to be scheduled onto nodes with matching taints. 
 
-| Name                             | Description                              | Default  |
-| -------------------------------- | ---------------------------------------- | -------  |
-| postgres_image                   | Path of the image to pull                | postgres |
-| postgres_image_version           | Image version to pull                    | 13       |
-| node_selector                    | AWX pods' nodeSelector                   | ''       |
-| web_node_selector                | AWX web pods' nodeSelector               | ''       |
-| task_node_selector               | AWX task pods' nodeSelector              | ''       |
-| topology_spread_constraints      | AWX pods' topologySpreadConstraints      | ''       |
-| web_topology_spread_constraints  | AWX web pods' topologySpreadConstraints  | ''       |
-| task_topology_spread_constraints | AWX task pods' topologySpreadConstraints | ''       |
-| affinity                         | AWX pods' affinity rules                 | ''       |
-| web_affinity                     | AWX web pods' affinity rules             | ''       |
-| task_affinity                    | AWX task pods' affinity rules            | ''       |
-| tolerations                      | AWX pods' tolerations                    | ''       |
-| web_tolerations                  | AWX web pods' tolerations                | ''       |
-| task_tolerations                 | AWX task pods' tolerations               | ''       |
-| annotations                      | AWX pods' annotations                    | ''       |
-| postgres_selector                | Postgres pods' nodeSelector              | ''       |
-| postgres_tolerations             | Postgres pods' tolerations               | ''       |
+| Name                             | Description                              | Default                          |
+| -------------------------------- | ---------------------------------------- | -------------------------------- |
+| postgres_image                   | Path of the image to pull                | quay.io/sclorg/postgresql-15-c9s |
+| postgres_image_version           | Image version to pull                    | latest                           |
+| node_selector                    | AWX pods' nodeSelector                   | ''                               |
+| web_node_selector                | AWX web pods' nodeSelector               | ''                               |
+| task_node_selector               | AWX task pods' nodeSelector              | ''                               |
+| topology_spread_constraints      | AWX pods' topologySpreadConstraints      | ''                               |
+| web_topology_spread_constraints  | AWX web pods' topologySpreadConstraints  | ''                               |
+| task_topology_spread_constraints | AWX task pods' topologySpreadConstraints | ''                               |
+| affinity                         | AWX pods' affinity rules                 | ''                               |
+| web_affinity                     | AWX web pods' affinity rules             | ''                               |
+| task_affinity                    | AWX task pods' affinity rules            | ''                               |
+| tolerations                      | AWX pods' tolerations                    | ''                               |
+| web_tolerations                  | AWX web pods' tolerations                | ''                               |
+| task_tolerations                 | AWX task pods' tolerations               | ''                               |
+| annotations                      | AWX pods' annotations                    | ''                               |
+| postgres_selector                | Postgres pods' nodeSelector              | ''                               |
+| postgres_tolerations             | Postgres pods' tolerations               | ''                               |
 
 Example of customization could be:
 
@@ -88,3 +88,8 @@ spec:
               - S2
           topologyKey: topology.kubernetes.io/zone
 ```
+
+#### Special Note on DB-Migration Job Scheduling
+
+For the **db-migration job**, which applies database migrations at cluster startup, you can specify scheduling settings using the `task_*` configurations such as `task_node_selector`, `task_tolerations`, etc.  
+If these task-specific settings are not defined, the job will automatically use the global AWX configurations like `node_selector` and `tolerations`.

docs/user-guide/advanced-configuration/container-probes.md

@@ -0,0 +1,52 @@
+#### Container Probes
+These parameters control the usage of liveness and readiness container probes for
+the web and task containers.
+
+> [!ALERT]
+> All of probes are disabled by default for now, to enable it, set the *_period parameters.  For example:
+
+```
+
+web_liveness_period: 15
+web_readiness_period: 15
+task_liveness_period: 15
+task_readiness_period: 15
+```
+
+#### Web / Task Container Liveness Check
+
+The liveness probe queries the status of the supervisor daemon of the container.  The probe will fail if it
+detects one of the services in a state other than "RUNNING".
+
+| Name         | Description                        | Default |
+| -------------| -----------------------------------|---------|
+| web_liveness_period | Time period in seconds between each probe check.  The value of 0 disables the probe. | 0    |
+| web_liveness_initial_delay | Initial delay before starting probes in seconds | 5   |
+| web_liveness_failure_threshold| Number of consecutive failure events to identify failure of container | 3    |
+| web_liveness_timeout | Number of seconds to wait for a probe response from container | 1    |
+| task_liveness_period | Time period in seconds between each probe check.  The value of 0 disables the probe. | 0    |
+| task_liveness_initial_delay | Initial delay before starting probes in seconds | 5   |
+| task_liveness_failure_threshold| Number of consecutive failure events to identify failure of container | 3    |
+| task_liveness_timeout | Number of seconds to wait for a probe response from container | 1    |
+
+#### Web Container Readiness Check
+
+This is an HTTP check against the status endpoint to confirm the system is still able to respond to web requests.
+
+| Name         | Description                        | Default |
+| -------------| ---------------------------------- | ------- |
+| web_readiness_period | Time period in seconds between each probe check.  The value of 0 disables the probe. | 0    |
+| web_readiness_initial_delay | Initial delay before starting probes in seconds | 5   |
+| web_readiness_failure_threshold| Number of consecutive failure events to identify failure of container | 3    |
+| web_readiness_timeout | Number of seconds to wait for a probe response from container | 1    |
+
+#### Task Container Readiness Check
+
+This is a command probe using the builtin check command of the awx-manage utility.
+
+| Name         | Description                        | Default |
+| -------------| ---------------------------------- | ------- |
+| task_readiness_period | Time period in seconds between each probe check.  The value of 0 disables the probe. | 0    |
+| task_readiness_initial_delay | Initial delay before starting probes in seconds | 5   |
+| task_readiness_failure_threshold| Number of consecutive failure events to identify failure of container | 3    |
+| task_readiness_timeout | Number of seconds to wait for a probe response from container | 1    |

docs/user-guide/advanced-configuration/containers-resource-requirements.md

@@ -22,11 +22,16 @@ spec:
 
 The resource requirements for both, the task and the web containers are configurable - both the lower end (requests) and the upper end (limits).
 
-| Name                       | Description                                      | Default                              |
-| -------------------------- | ------------------------------------------------ | ------------------------------------ |
-| web_resource_requirements  | Web container resource requirements              | requests: {cpu: 100m, memory: 128Mi} |
-| task_resource_requirements | Task container resource requirements             | requests: {cpu: 100m, memory: 128Mi} |
-| ee_resource_requirements   | EE control plane container resource requirements | requests: {cpu: 100m, memory: 128Mi} |
+| Name                                 | Description                                                  | Default                              |
+| ------------------------------------ | ------------------------------------------------------------ | ------------------------------------ |
+| web_resource_requirements            | Web container resource requirements                          | requests: {cpu: 100m, memory: 128Mi} |
+| task_resource_requirements           | Task container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
+| ee_resource_requirements             | EE control plane container resource requirements             | requests: {cpu: 50m, memory: 64Mi} |
+| redis_resource_requirements          | Redis container resource requirements                        | requests: {cpu: 100m, memory: 128Mi} |
+| postgres_resource_requirements       | Postgres container (and initContainer) resource requirements | requests: {cpu: 10m, memory: 64Mi}   |
+| rsyslog_resource_requirements        | Rsyslog container resource requirements                      | requests: {cpu: 100m, memory: 128Mi} |
+| init_container_resource_requirements | Init Container resource requirements                         | requests: {cpu: 100m, memory: 128Mi} |
+
 
 Example of customization could be:
 
@@ -34,31 +39,56 @@ Example of customization could be:
 ---
 spec:
   ...
-  web_resource_requirements:
-    requests:
-      cpu: 250m
-      memory: 2Gi
-      ephemeral-storage: 100M
-    limits:
-      cpu: 1000m
-      memory: 4Gi
-      ephemeral-storage: 500M
+
   task_resource_requirements:
     requests:
-      cpu: 250m
-      memory: 1Gi
+      cpu: 100m
+      memory: 128Mi
       ephemeral-storage: 100M
     limits:
       cpu: 2000m
-      memory: 2Gi
+      memory: 4Gi
       ephemeral-storage: 500M
+  web_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 1000m
+      memory: 4Gi
   ee_resource_requirements:
     requests:
-      cpu: 250m
-      memory: 100Mi
-      ephemeral-storage: 100M
+      cpu: 100m
+      memory: 64Mi
     limits:
-      cpu: 500m
+      cpu: 1000m
+      memory: 4Gi
+  redis_resource_requirements:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 1000m
+      memory: 4Gi
+  rsyslog_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 1000m
+      memory: 2Gi
+  init_container_resource_requirements:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 1000m
       memory: 2Gi
-      ephemeral-storage: 500M
 ```
+
+
+#### Limits and ResourceQuotas
+
+If the cluster you are deploying in has a ResoruceQuota, you will need to configure resource limits for all of the pods deployed in that cluster. This can be done for AWX pods on the AWX spec in the manner shown above.
+
+There is an example you can use in [`awx_v1beta1_awx_resource_limits.yaml`](https://raw.githubusercontent.com/ansible/awx-operator/devel/config/samples/awx_v1beta1_awx_resource_limits.yaml).

docs/user-guide/advanced-configuration/custom-receptor-certs.md

@@ -0,0 +1,24 @@
+
+### Custom Receptor CA
+
+The control nodes on the K8S cluster will communicate with execution nodes via mutual TLS TCP connections, running via Receptor.
+Execution nodes will verify incoming connections by ensuring the x509 certificate was issued by a trusted Certificate Authority (CA).
+
+A user may wish to provide their own CA for this validation. If no CA is provided, AWX Operator will automatically generate one using OpenSSL.
+
+Given custom `ca.crt` and `ca.key` stored locally, run the following,
+
+```bash
+kubectl create secret tls awx-demo-receptor-ca \
+   --cert=/path/to/ca.crt --key=/path/to/ca.key
+```
+
+The secret should be named `{AWX Custom Resource name}-receptor-ca`. In the above the AWX CR name is "awx-demo". Please replace "awx-demo" with your AWX Custom Resource name.
+
+If this secret is created after AWX is deployed, run the following to restart the deployment,
+
+```bash
+kubectl rollout restart deployment awx-demo
+```
+
+**Important Note**, changing the receptor CA will break connections to any existing execution nodes. These nodes will enter an `unavailable` state, and jobs will not be able to run on them. Users will need to download and re-run the install bundle for each execution node. This will replace the TLS certificate files with those signed by the new CA. The execution nodes should then appear in a `ready` state after a few minutes.

docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md

@@ -1,4 +1,4 @@
-#### Custom Volume and Volume Mount Options
+# Custom Volume and Volume Mount Options
 
 In a scenario where custom volumes and volume mounts are required to either overwrite defaults or mount configuration files.
 
@@ -12,8 +12,8 @@ In a scenario where custom volumes and volume mounts are required to either over
 | init_container_extra_volume_mounts | Specify volume mounts to be added to Init container      | ''      |
 | init_container_extra_commands      | Specify additional commands for Init container           | ''      |
 
-
-> :warning: The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
+!!! warning
+    The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec).
 
 Example configuration for ConfigMap
 
@@ -26,64 +26,50 @@ metadata:
   namespace: <target namespace>
 data:
   ansible.cfg: |
-     [defaults]
-     remote_tmp = /tmp
-     [ssh_connection]
-     ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
-  custom.py:  |
-      INSIGHTS_URL_BASE = "example.org"
-      AWX_CLEANUP_PATHS = True
+    [defaults]
+    remote_tmp = /tmp
+    [ssh_connection]
+    ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s
 ```
+
 Example spec file for volumes and volume mounts
 
 ```yaml
 ---
-    spec:
-    ...
-      extra_volumes: |
-        - name: ansible-cfg
-          configMap:
-            defaultMode: 420
-            items:
-              - key: ansible.cfg
-                path: ansible.cfg
-            name: <resourcename>-extra-config
-        - name: custom-py
-          configMap:
-            defaultMode: 420
-            items:
-              - key: custom.py
-                path: custom.py
-            name: <resourcename>-extra-config
-        - name: shared-volume
-          persistentVolumeClaim:
-            claimName: my-external-volume-claim
+spec:
+  ...
+  extra_volumes: |
+    - name: ansible-cfg
+      configMap:
+        defaultMode: 420
+        items:
+          - key: ansible.cfg
+            path: ansible.cfg
+        name: <resourcename>-extra-config
+    - name: shared-volume
+      persistentVolumeClaim:
+        claimName: my-external-volume-claim
 
-      init_container_extra_volume_mounts: |
-        - name: shared-volume
-          mountPath: /shared
+  init_container_extra_volume_mounts: |
+    - name: shared-volume
+      mountPath: /shared
 
-      init_container_extra_commands: |
-        # set proper permissions (rwx) for the awx user
-        chmod 775 /shared
-        chgrp 1000 /shared
+  init_container_extra_commands: |
+    # set proper permissions (rwx) for the awx user
+    chmod 775 /shared
+    chgrp 1000 /shared
 
-      ee_extra_volume_mounts: |
-        - name: ansible-cfg
-          mountPath: /etc/ansible/ansible.cfg
-          subPath: ansible.cfg
-
-      task_extra_volume_mounts: |
-        - name: custom-py
-          mountPath: /etc/tower/conf.d/custom.py
-          subPath: custom.py
-        - name: shared-volume
-          mountPath: /shared
+  ee_extra_volume_mounts: |
+    - name: ansible-cfg
+      mountPath: /etc/ansible/ansible.cfg
+      subPath: ansible.cfg
 ```
 
-> :warning: **Volume and VolumeMount names cannot contain underscores(_)**
+!!! warning
+    **Volume and VolumeMount names cannot contain underscores(_)**
+
+## Custom UWSGI Configuration
 
-##### Custom UWSGI Configuration
 We allow the customization of two UWSGI parameters:
 
 * [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5)
@@ -103,7 +89,7 @@ requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related nginx
 configuration.
 
-##### Custom Nginx Configuration
+## Custom Nginx Configuration
 
 Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf.
 
@@ -124,26 +110,70 @@ may allow the web pods to handle more "bursty" request patterns if many
 requests (more than 128) tend to come in a short period of time, but can all be
 handled before any other time outs may apply. Also see related uwsgi
 configuration.
+
 * [worker_processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes) with `nginx_worker_processes` (default of 1)
 * [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) with `nginx_worker_cpu_affinity` (default "auto")
 * [worker_connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) with `nginx_worker_connections` (minimum of 1024)
 * [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size)
 
+## Custom Logos
 
-##### Custom Favicon
+You can use custom volume mounts to mount in your own logos to be displayed instead of the AWX logo.
+There are two different logos, one to be displayed on page headers, and one for the login screen.
 
-You can use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
-
-First, Create the configmap from a local favicon.ico file.
+First, create configmaps for the logos from local `logo-login.svg` and `logo-header.svg` files.
 
 ```bash
-$ oc create configmap favicon-configmap --from-file favicon.ico
+kubectl create configmap logo-login-configmap --from-file logo-login.svg
+kubectl create configmap logo-header-configmap --from-file logo-header.svg
 ```
 
 Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
 
 ```yaml
+---
 spec:
+  ...
+  extra_volumes: |
+    - name: logo-login
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-login.svg
+            path: logo-login.svg
+        name: logo-login-configmap
+    - name: logo-header
+      configMap:
+        defaultMode: 420
+        items:
+          - key: logo-header.svg
+            path: logo-header.svg
+        name: logo-header-configmap
+  web_extra_volume_mounts: |
+    - name: logo-login
+      mountPath: /var/lib/awx/public/static/media/logo-login.svg
+      subPath: logo-login.svg
+    - name: logo-header
+      mountPath: /var/lib/awx/public/static/media/logo-header.svg
+      subPath: logo-header.svg
+```
+
+## Custom Favicon
+
+You can also use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab.
+
+First, create the configmap from a local `favicon.ico` file.
+
+```bash
+kubectl create configmap favicon-configmap --from-file favicon.ico
+```
+
+Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec
+
+```yaml
+---
+spec:
+  ...
   extra_volumes: |
     - name: favicon
       configMap:
@@ -157,3 +187,7 @@ spec:
       mountPath: /var/lib/awx/public/static/media/favicon.ico
       subPath: favicon.ico
 ```
+
+## Custom AWX Configuration
+
+Refer to the [Extra Settings](./extra-settings.md) documentation for customizing the AWX configuration.

docs/user-guide/advanced-configuration/deploying-a-specific-version-of-awx.md

@@ -2,15 +2,19 @@
 
 There are a few variables that are customizable for awx the image management.
 
-| Name                | Description               | Default                                 |
-| ------------------- | ------------------------- | --------------------------------------  |
-| image               | Path of the image to pull | quay.io/ansible/awx                     |
-| image_version       | Image version to pull     | value of DEFAULT_AWX_VERSION or latest  |
-| image_pull_policy   | The pull policy to adopt  | IfNotPresent                            |
-| image_pull_secrets  | The pull secrets to use   | None                                    |
-| ee_images           | A list of EEs to register | quay.io/ansible/awx-ee:latest           |
-| redis_image         | Path of the image to pull | docker.io/redis                         |
-| redis_image_version | Image version to pull     | latest                                  |
+| Name                          | Description               | Default                                    |
+| ----------------------------- | ------------------------- | ------------------------------------------ |
+| image                         | Path of the image to pull | quay.io/ansible/awx                        |
+| image_version                 | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| image_pull_policy             | The pull policy to adopt  | IfNotPresent                               |
+| image_pull_secrets            | The pull secrets to use   | None                                       |
+| ee_images                     | A list of EEs to register | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| redis_image                   | Path of the image to pull | docker.io/redis                            |
+| redis_image_version           | Image version to pull     | latest                                     |
+| control_plane_ee_image        | Image version to pull     | quay.io/ansible/awx-ee:DEFAULT_AWX_VERSION |
+| init_container_image          | Path of the image to pull | quay.io/ansible/awx-ee                     |
+| init_container_image_version  | Image version to pull     | value of DEFAULT_AWX_VERSION or latest     |
+| init_projects_container_image | Image version to pull     | quay.io/centos/centos:stream9              |
 
 Example of customization could be:
 
@@ -26,6 +30,10 @@ spec:
   ee_images:
     - name: my-custom-awx-ee
       image: myorg/my-custom-awx-ee
+  control_plane_ee_image: myorg/my-custom-awx-ee:latest
+  init_container_image: myorg/my-custom-awx-ee
+  init_container_image_version: latest
+  init_projects_container_image: myorg/my-mirrored-centos:stream9
 ```
 
 **Note**: The `image` and `image_version` are intended for local mirroring scenarios. Please note that using a version of AWX other than the one bundled with the `awx-operator` is **not** supported. For the default values, check the [main.yml](https://github.com/ansible/awx-operator/blob/devel/roles/installer/defaults/main.yml) file.

docs/user-guide/advanced-configuration/disable-ipv6.md

@@ -16,30 +16,3 @@ The following variables are customizable
 spec:
   ipv6_disabled: true
 ```
-
-### Adding Execution Nodes
-Starting with AWX Operator v0.30.0 and AWX v21.7.0, standalone execution nodes can be added to your deployments.
-See [AWX execution nodes docs](https://github.com/ansible/awx/blob/devel/docs/execution_nodes.md) for information about this feature.
-
-#### Custom Receptor CA
-The control nodes on the K8S cluster will communicate with execution nodes via mutual TLS TCP connections, running via Receptor.
-Execution nodes will verify incoming connections by ensuring the x509 certificate was issued by a trusted Certificate Authority (CA).
-
-A user may wish to provide their own CA for this validation. If no CA is provided, AWX Operator will automatically generate one using OpenSSL.
-
-Given custom `ca.crt` and `ca.key` stored locally, run the following,
-
-```bash
-kubectl create secret tls awx-demo-receptor-ca \
-   --cert=/path/to/ca.crt --key=/path/to/ca.key
-```
-
-The secret should be named `{AWX Custom Resource name}-receptor-ca`. In the above the AWX CR name is "awx-demo". Please replace "awx-demo" with your AWX Custom Resource name.
-
-If this secret is created after AWX is deployed, run the following to restart the deployment,
-
-```bash
-kubectl rollout restart deployment awx-demo
-```
-
-**Important Note**, changing the receptor CA will break connections to any existing execution nodes. These nodes will enter an `unavailable` state, and jobs will not be able to run on them. Users will need to download and re-run the install bundle for each execution node. This will replace the TLS certificate files with those signed by the new CA. The execution nodes should then appear in a `ready` state after a few minutes.

docs/user-guide/advanced-configuration/extra-settings.md

@@ -1,26 +1,119 @@
-#### Extra Settings
+# Extra Settings
 
-With`extra_settings`, you can pass multiple custom settings via the `awx-operator`. The parameter `extra_settings`  will be appended to the `/etc/tower/settings.py` and can be an alternative to the `extra_volumes` parameter.
+With `extra_settings` and `extra_settings_files`, you can pass multiple custom settings to AWX via the AWX Operator.
 
-| Name           | Description    | Default |
-| -------------- | -------------- | ------- |
-| extra_settings | Extra settings | ''      |
+!!! note
+    Parameters configured in `extra_settings` or `extra_settings_files` are set as read-only settings in AWX. As a result, they cannot be changed in the UI after deployment.
 
-**Note:** Parameters configured in `extra_settings` are set as read-only settings in AWX.  As a result, they cannot be changed in the UI after deployment. If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec.  
+    If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec (for `extra_settings`) or corresponding ConfigMap or Secret (for `extra_settings_files`). After updating ConfigMap or Secret, you need to restart the AWX pods to apply the changes.
+
+!!! note
+    If the same setting is set in both `extra_settings` and `extra_settings_files`, the setting in `extra_settings_files` will take precedence.
+
+## Add extra settings with `extra_settings`
+
+You can pass extra settings by specifying the pair of the setting name and value as the `extra_settings` parameter.
+
+The settings passed via `extra_settings` will be appended to the `/etc/tower/settings.py`.
+
+| Name           | Description    | Default   |
+| -------------- | -------------- | --------- |
+| extra_settings | Extra settings | `[]`      |
 
 Example configuration of `extra_settings` parameter
 
 ```yaml
-  spec:
-    extra_settings:
-      - setting: MAX_PAGE_SIZE
-        value: "500"
+spec:
+  extra_settings:
+    - setting: MAX_PAGE_SIZE
+      value: "500"
 
-      - setting: AUTH_LDAP_BIND_DN
-        value: "cn=admin,dc=example,dc=com"
+    - setting: AUTH_LDAP_BIND_DN
+      value: "cn=admin,dc=example,dc=com"
 
-      - setting: LOG_AGGREGATOR_LEVEL
-        value: "'DEBUG'"
+    - setting: LOG_AGGREGATOR_LEVEL
+      value: "'DEBUG'"
 ```
 
 Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes.
+
+## Add extra settings with `extra_settings_files`
+
+You can pass extra settings by specifying the additional settings files in the ConfigMaps or Secrets as the `extra_settings_files` parameter.
+
+The settings files passed via `extra_settings_files` will be mounted as the files under the `/etc/tower/conf.d`.
+
+| Name                 | Description          | Default   |
+| -------------------- | -------------------- | --------- |
+| extra_settings_files | Extra settings files | `{}`      |
+
+!!! note
+    If the same setting is set in multiple files in `extra_settings_files`, it would be difficult to predict which would be adopted since these files are loaded in arbitrary order that [`glob`](https://docs.python.org/3/library/glob.html) returns. For a reliable setting, do not include the same key in more than one file.
+
+Create ConfigMaps or Secrets that contain custom settings files (`*.py`).
+
+```python title="custom_job_settings.py"
+AWX_TASK_ENV = {
+    "HTTPS_PROXY": "http://proxy.example.com:3128",
+    "HTTP_PROXY": "http://proxy.example.com:3128",
+    "NO_PROXY": "127.0.0.1,localhost,.example.com"
+}
+GALAXY_TASK_ENV = {
+    "ANSIBLE_FORCE_COLOR": "false",
+    "GIT_SSH_COMMAND": "ssh -o StrictHostKeyChecking=no",
+}
+```
+
+```python title="custom_system_settings.py"
+REMOTE_HOST_HEADERS = [
+    "HTTP_X_FORWARDED_FOR",
+    "REMOTE_ADDR",
+    "REMOTE_HOST",
+]
+```
+
+```python title="custom_passwords.py"
+SUBSCRIPTIONS_PASSWORD = "my-super-secure-subscription-password123!"
+REDHAT_PASSWORD = "my-super-secure-redhat-password123!"
+```
+
+```bash title="Create ConfigMap and Secret"
+# Create ConfigMap
+kubectl create configmap my-custom-settings \
+  --from-file /PATH/TO/YOUR/custom_job_settings.py \
+  --from-file /PATH/TO/YOUR/custom_system_settings.py
+
+# Create Secret
+kubectl create secret generic my-custom-passwords \
+  --from-file /PATH/TO/YOUR/custom_passwords.py
+```
+
+Then specify them in the AWX CR spec. Here is an example configuration of `extra_settings_files` parameter.
+
+```yaml
+spec:
+  extra_settings_files:
+    configmaps:
+      - name: my-custom-settings        # The name of the ConfigMap
+        key: custom_job_settings.py     # The key in the ConfigMap, which means the file name
+      - name: my-custom-settings
+        key: custom_system_settings.py
+    secrets:
+      - name: my-custom-passwords       # The name of the Secret
+        key: custom_passwords.py        # The key in the Secret, which means the file name
+```
+
+!!! Warning "Restriction"
+    There are some restrictions on the ConfigMaps or Secrets used in `extra_settings_files`.
+
+    - The keys in ConfigMaps or Secrets MUST be the name of python files and MUST end with `.py`
+    - The keys in ConfigMaps or Secrets MUST consists of alphanumeric characters, `-`, `_` or `.`
+    - The keys in ConfigMaps or Secrets are converted to the following strings, which MUST not exceed 63 characters
+        - Keys in ConfigMaps: `<instance name>-<KEY>-configmap`
+        - Keys in Secrets: `<instance name>-<KEY>-secret`
+    - Following keys are reserved and MUST NOT be used in ConfigMaps or Secrets
+        - `credentials.py`
+        - `execution_environments.py`
+        - `ldap.py`
+
+    Refer to the Kubernetes documentations ([[1]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/config-map-v1/), [[2]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/), [[3]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/), [[4]](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)) for more information about character types and length restrictions.

docs/user-guide/advanced-configuration/horizontal-pod-autoscaler.md

@@ -0,0 +1,27 @@
+### Horizontal Pod Autoscaler (HPA)
+
+Horizontal Pod Autoscaler allows Kubernetes to scale the number of replicas of
+deployments in response to configured metrics.
+
+This feature conflicts with the operators ability to manage the number of static
+replicas to create for each deployment.
+
+The use of the settings below will tell the operator to not manage the replicas
+field on the identified deployments even if a replicas count has been set for those
+properties in the operator resource.
+
+| Name                   | Description                               | Default |
+| -----------------------| ----------------------------------------- | ------- |
+| web_manage_replicas    | Indicates operator should control the     | true    |
+|                        | replicas count for the web deployment.    |         |
+|                        |                                           |         |
+| task_manage_replicas   | Indicates operator should control the     | true    |
+|                        | replicas count for the task deployment.   |         |
+
+#### Recommended Settings for HPA
+
+Please see the Kubernetes documentation on how to configure the horizontal pod
+autoscaler.
+
+The values for optimal HPA are cluster and need specific so general guidelines
+are not available at this time.

docs/user-guide/advanced-configuration/mesh-ingress.md

@@ -0,0 +1,229 @@
+# Mesh Ingress
+
+The mesh ingress allows users to peer external execution and hop nodes into the AWX control plane.
+This guide focuses on how to enable and configure the mesh ingress.
+For more information about remote execution and hop nodes and how to create them, refer to the [Managing Capacity With Instances](https://ansible.readthedocs.io/projects/awx/en/latest/administration/instances.html) chapter of the AWX Administration Guide.
+
+## Prerequisites
+
+- AWX operator version > 2.11.0
+- AWX > 23.8.0
+
+## Deploy and configure AWXMeshIngress
+
+!!! note
+    The mesh ingress uses the `control_plane_ee_image` and `image_pull_policy` fields of the AWX instance to determine image and policy to be adopted.
+    Defaulted to `quay.io/ansible/awx-ee:latest` and `Always`.
+    Currently there are no dedicated parameters to specify the image and policy.
+
+### On Red Hat OpenShift with Operator managed Route
+
+To deploy a mesh ingress on OpenShift, create the AWXMeshIngress resource on the namespace where your AWX instance is running on.
+
+Example:
+
+```yaml
+---
+apiVersion: awx.ansible.com/v1alpha1
+kind: AWXMeshIngress
+metadata:
+  name: <mesh ingress name>
+spec:
+  deployment_name: <awx instance name>
+```
+
+### On Kubernetes with Operator managed Ingress (NGINX)
+
+To deploy a mesh ingress on Kubernetes cluster which has [NGINX Ingress Controller](https://www.nginx.com/products/nginx-ingress-controller/), create the AWXMeshIngress resource on the namespace where your AWX instance is running on.
+
+Note that AWXMeshIngress requires [SSL Passthrough](https://kubernetes.github.io/ingress-nginx/user-guide/tls/#ssl-passthrough) enabled which is disabled by default. Ensure it is enabled on your NGINX Ingress Controller.
+
+By specifying `ingress_controller` as `nginx`, AWX Operator will generate Ingress resource that has `nginx.ingress.kubernetes.io/ssl-passthrough` annotation set to `"true"`.
+
+Example:
+
+```yaml
+---
+apiVersion: awx.ansible.com/v1alpha1
+kind: AWXMeshIngress
+metadata:
+  name: <mesh ingress name>
+spec:
+  deployment_name: <awx instance name>
+
+  ingress_type: Ingress
+  ingress_controller: nginx
+  ingress_class_name: nginx
+
+  external_hostname: <fqdn for mesh ingress>
+```
+
+### On Kubernetes with Operator managed Ingress (Traefik)
+
+To deploy a mesh ingress on Kubernetes cluster which has [Traefik Kubernetes Ingress provider](https://doc.traefik.io/traefik/providers/kubernetes-ingress/), create the AWXMeshIngress resource on the namespace where your AWX instance is running on.
+
+Note that by deploying following AWXMeshIngress, AWX Operator will generate IngressRouteTCP resource that has `websecure` as an `entryPoints`. If this does not satisfy your requirement, refer to [User managed Ingress section](#on-kubernetes-with-user-managed-ingress) and  create an IngressRouteTCP resource manually.
+
+Example:
+
+```yaml
+---
+apiVersion: awx.ansible.com/v1alpha1
+kind: AWXMeshIngress
+metadata:
+  name: <mesh ingress name>
+spec:
+  deployment_name: <awx instance name>
+
+  ingress_type: IngressRouteTCP
+  ingress_controller: traefik
+  ingress_class_name: traefik
+  ingress_api_version: traefik.io/v1alpha1
+
+  external_hostname: <fqdn for mesh ingress>
+```
+
+!!! tip
+    AWX Operator supports both API groups `traefik.io` and `traefik.containo.us` in `ingress_api_version` for Traefik, but it is recommended to use `traefik.io` since `traefik.containo.us` is deprecated in Traefik v2.10 and is removed in Traefik v3. Refer to [Traefik documentation](https://doc.traefik.io/traefik/migration/v2/#v210) for more information about deprecation.
+
+    If you can't see any IngressRouteTCP resources by `kubectl` command after deploying mesh ingress, you should fully qualify the resource name with API group, `kubectl get ingressroutetcp.traefik.io` or `kubectl get ingressroutetcp.traefik.containo.us` for example.
+
+### On Kubernetes with User managed Ingress
+
+To deploy a mesh ingress on Kubernetes cluster, create the AWXMeshIngress resource on the namespace where your AWX instance is running on.
+
+Alternatively, if you wish to create your own Ingress resource, you can deploy a mesh ingress with `ingress_type` set to `none` and then manually create an Ingress resource with any configuration.
+
+In this case, the `external_hostname` is still required as it is used to generate the certificate that will be used by Receptor.
+
+Example:
+
+```yaml
+---
+apiVersion: awx.ansible.com/v1alpha1
+kind: AWXMeshIngress
+metadata:
+  name: <mesh ingress name>
+spec:
+  deployment_name: <awx instance name>
+
+  ingress_type: none  # This line can be omitted since this is the default value
+  external_hostname: <fqdn for mesh ingress>
+```
+
+The requirements for user managed Ingress resource are as follows:
+
+- Supports WebSocket
+- SSL/TLS Passthrough enabled
+- Accessible over port `443`
+- Having the same hostname as `external_hostname` in the AWXMeshIngress resource
+- Routing the traffic to port `27199` of the Service of the same name as the AWXMeshIngress resource
+
+These are example Ingress resources for NGINX and Traefik.
+
+```yaml
+# Ingress for NGINX Ingress Controller
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: <mesh ingress name>
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: <fqdn for mesh ingress>
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: <mesh ingress name>
+                port:
+                  number: 27199
+```
+
+```yaml
+# Ingress for Traefik Kubernetes Ingress provider
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRouteTCP
+metadata:
+  name: <mesh ingress name>
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: HostSNI(`<fqdn for mesh ingress>`)
+      services:
+        - name: <mesh ingress name>
+          port: 27199
+  tls:
+    passthrough: true
+```
+
+## Validating setup of Mesh Ingress
+
+After AWXMeshIngress has been successfully created, a new Instance with the same name will be registered to AWX and will be visible on the Instance UI page
+
+![mesh ingress instance on AWX UI](mesh-ingress-instance-on-awx-ui.png)
+
+The Instance should have at least 2 listener addresses.
+
+In this example, the mesh ingress has two listener addresses:
+
+- one for internal, that is used for peering to by all control nodes (top)
+- one for external, that is exposed to a route so external execution nodes can peer into it (bottom))
+
+![mesh ingress instance listener address on awx ui](mesh-ingress-instance-listener-address-on-awx-ui.png)
+
+When selecting peer for new instance the mesh ingress instance should now be present as a option.
+![peering to mesh ingress on awx ui](peering-to-mesh-ingress-on-awx-ui.png)
+
+For more information about how to create external remote execution and hop nodes and configuring the mesh, see AWX Documentation on [Add a instance](https://ansible.readthedocs.io/projects/awx/en/latest/administration/instances.html#add-an-instance).
+
+## Custom Resource Definitions
+
+### AWXMeshIngress
+
+AWXMeshIngress controls the deployment and configuration of mesh ingress on AWX
+
+| Name                                                                                                                          | Description                                                                                                                                                         |
+| ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **`apiVersion`**                                                                                                              | awx.ansible.com/v1alpha1                                                                                                                                            |
+| **`kind`**                                                                                                                    | AWXMeshIngress                                                                                                                                                      |
+| **`metadata`** ([ObjectMeta](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/object-meta/#ObjectMeta)) | Standard object's metadata. [More info](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata)                               |
+| **`spec`** ([AWXMeshIngressSpec](#awxmeshingressspec))                                                                        | Spec is the desired state of the AWXMeshIngress. [More info](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status)   |
+| **`status`** ([AWXMeshIngressStatus](#awxmeshingressstatus))                                                                  | Status is the current state of the AWXMeshIngress. [More info](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status) |
+
+#### AWXMeshIngressSpec
+
+AWXMeshIngressSpec is the description of the configuration for AWXMeshIngress.
+
+| Name                                     | Description                                                                                                                                                                                                                                 | Default                                        |
+| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- |
+| **`deployment_name`** (string), required | Name of the AWX deployment to create the Mesh Ingress for.                                                                                                                                                                                  | `awx`                                          |
+| **`ingress_type`** (string)              | Ingress type for ingress managed by the operator. Options: `none`, `Ingress`, `IngressRouteTCP`, `Route`                                                                                                                                    | `Route` (on OpenShift), `none` (on Kubernetes) |
+| **`external_hostname`** (string)         | External hostname is an optional field used for specifying the external hostname defined in an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/). This parameter is automatically generated on OpenShift          | N/A                                            |
+| **`external_ipaddress`** (string)        | External IP Address is an optional field used for specifying the external IP address defined in an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/)                                                              | N/A                                            |
+| **`ingress_api_version`** (string)       | API Version for ingress managed by the operator. This parameter is ignored when `ingress_type` is `Route`                                                                                                                                   | `networking.k8s.io/v1`                         |
+| **`ingress_annotations`** (string)       | Additional annotation on the ingress managed by the operator. This parameter is ignored when `ingress_type` is `Route`                                                                                                                      | `""`                                           |
+| **`ingress_controller`** (string)        | Special configuration for specific Ingress Controllers. This parameter is ignored when `ingress_type` is `Route`                                                                                                                            | `""`                                           |
+| **`ingress_class_name`** (string)        | The name of ingress class to use instead of the cluster default. see [IngressSpec](https://kubernetes.io/docs/reference/kubernetes-api/service-resources/ingress-v1/#IngressSpec). This parameter is ignored when `ingress_type` is `Route` | `""`                                           |
+
+#### AWXMeshIngressStatus
+
+AWXMeshIngressStatus describe the current state of the AWXMeshIngress.
+
+### AWXMeshIngressList
+
+AWXMeshIngressList is a collection of AWXMeshIngress.
+
+| Name                                                                                                                    | Description                                                                                                                                                                                                                                                                                          |
+| ----------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **`items`** ([AWXMeshIngress](#awxmeshingress))                                                                         | items is the list of Ingress.                                                                                                                                                                                                                                                                        |
+| **`apiVersion`** (string)                                                                                               | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. [More info](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources)  |
+| **`kind`** (string)                                                                                                     | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. [More info](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds) |
+| **`metadata`** ([ListMeta](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/list-meta/#ListMeta)) | Standard object's metadata. [More info](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata)                                                                                                                                                                |

docs/user-guide/advanced-configuration/scaling-the-web-and-task-pods-independently.md

@@ -1,8 +1,13 @@
-#### Scaling the Web and Task Pods independently 
+#### Scaling the Web and Task Pods independently
 
 You can scale replicas up or down for each deployment by using the `web_replicas` or `task_replicas` respectively. You can scale all pods across both deployments by using `replicas` as well. The logic behind these CRD keys acts as such:
 
-- If you specify the `replicas` field, the key passed will scale both the `web` and `task` replicas to the same number. 
+- If you specify the `replicas` field, the key passed will scale both the `web` and `task` replicas to the same number.
 - If `web_replicas` or `task_replicas` is ever passed, it will override the existing `replicas` field on the specific deployment with the new key value.
 
-These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found below in the [Assigning AWX pods to specific nodes](#assigning-awx-pods-to-specific-nodes) section. 
+These new replicas can be constrained in a similar manner to previous single deployments by appending the particular deployment name in front of the constraint used. More about those new constraints can be found in the [Assigning AWX pods to specific nodes](./assigning-awx-pods-to-specific-nodes.md) page.
+
+##### Horizontal Pod Autoscaling
+
+The operator is capable of working with Kubernete's HPA capabilities.  See [Horizontal Pod Autoscaler](./horizontal-pod-autoscaler.md)
+documentation for more information.

docs/user-guide/advanced-configuration/security-context.md

@@ -0,0 +1,27 @@
+#### Service Account
+
+It is possible to modify some `SecurityContext` proprieties of the various deployments and stateful sets if needed.
+
+| Name                               | Description                                  | Default |
+| ---------------------------------- | -------------------------------------------- | ------- |
+| security_context_settings          | SecurityContext for Task and Web deployments | {}      |
+| postgres_security_context_settings | SecurityContext for Task and Web deployments | {}      |
+
+
+Example configuration securityContext for the Task and Web deployments:
+
+```yaml
+spec:
+  security_context_settings:
+    allowPrivilegeEscalation: false
+    capabilities:
+        drop:
+        - ALL
+```
+
+
+```yaml
+spec:
+  postgres_security_context_settings:
+    runAsNonRoot: true
+```

docs/user-guide/database-configuration.md

@@ -1,10 +1,10 @@
 ### Database Configuration
 
-#### Postgres Version
+#### PostgreSQL Version
 
-The default Postgres version for the version of AWX bundled with the latest version of the awx-operator is Postgres 13. You can find this default for a given version by at the default value for [_postgres_image_version](https://github.com/ansible/awx-operator/blob/devel/roles/installer/defaults/main.yml#L243).
+The default PostgreSQL version for the version of AWX bundled with the latest version of the awx-operator is PostgreSQL 15. You can find this default for a given version by at the default value for [supported_pg_version](https://github.com/ansible/awx-operator/blob/ffba1b4712a0b03f1faedfa70e3a9ef0d443e4a6/roles/installer/vars/main.yml#L7).
 
-We only have coverage for the default version of Postgres. Newer versions of Postgres (14+) will likely work, but should only be configured as an external database. If your database is managed by the awx-operator (default if you don't specify a `postgres_configuration_secret`), then you should not override the default version as this may cause issues when awx-operator tries to upgrade your postgresql pod.
+We only have coverage for the default version of PostgreSQL. Newer versions of PostgreSQL will likely work, but should only be configured as an external database. If your database is managed by the awx-operator (default if you don't specify a `postgres_configuration_secret`), then you should not override the default version as this may cause issues when awx-operator tries to upgrade your postgresql pod.
 
 #### External PostgreSQL Service
 
@@ -27,6 +27,7 @@ stringData:
   username: <username to connect as>
   password: <password to connect with>
   sslmode: prefer
+  target_session_attrs: read-write
   type: unmanaged
 type: Opaque
 ```
@@ -37,6 +38,8 @@ type: Opaque
 
 **Note**: The variable `sslmode` is valid for `external` databases only. The allowed values are: `prefer`, `disable`, `allow`, `require`, `verify-ca`, `verify-full`.
 
+**Note**: The variable `target_session_attrs` is only useful for `clustered external` databases. The allowed values are: `any` (default), `read-write`, `read-only`, `primary`, `standby` and `prefer-standby`, whereby only `read-write` and `primary` really make sense in AWX use, as you want to connect to a database node that offers write support.
+
 Once the secret is created, you can specify it on your spec:
 
 ```yaml
@@ -56,15 +59,14 @@ If you don't have access to an external PostgreSQL service, the AWX operator can
 
 The following variables are customizable for the managed PostgreSQL service
 
-| Name                                          | Description                                   | Default                            |
-| --------------------------------------------- | --------------------------------------------- | ---------------------------------- |
-| postgres_image                                | Path of the image to pull                     | postgres:12                        |
-| postgres_init_container_resource_requirements | Database init container resource requirements | requests: {cpu: 10m, memory: 64Mi} |
-| postgres_resource_requirements                | PostgreSQL container resource requirements    | requests: {cpu: 10m, memory: 64Mi} |
-| postgres_storage_requirements                 | PostgreSQL container storage requirements     | requests: {storage: 8Gi}           |
-| postgres_storage_class                        | PostgreSQL PV storage class                   | Empty string                       |
-| postgres_data_path                            | PostgreSQL data path                          | `/var/lib/postgresql/data/pgdata`  |
-| postgres_priority_class                       | Priority class used for PostgreSQL pod        | Empty string                       |
+| Name                                          | Description                                                     | Default                                 |
+| --------------------------------------------- | --------------------------------------------------------------- | --------------------------------------- |
+| postgres_image                                | Path of the image to pull                                       | quay.io/sclorg/postgresql-15-c9s        |
+| postgres_image_version                        | Image version to pull                                           | latest                                  |
+| postgres_resource_requirements                | PostgreSQL container (and initContainer) resource requirements  | requests: {cpu: 10m, memory: 64Mi}      |
+| postgres_storage_requirements                 | PostgreSQL container storage requirements                       | requests: {storage: 8Gi}                |
+| postgres_storage_class                        | PostgreSQL PV storage class                                     | Empty string                            |
+| postgres_priority_class                       | Priority class used for PostgreSQL pod                          | Empty string                            |
 
 Example of customization could be:
 
@@ -90,4 +92,31 @@ spec:
     - 'max_connections=1000'
 ```
 
-**Note**: If `postgres_storage_class` is not defined, Postgres will store it's data on a volume using the default storage class for your cluster.
+**Note**: If `postgres_storage_class` is not defined, PostgreSQL will store it's data on a volume using the default storage class for your cluster.
+
+#### Note about overriding the postgres image
+
+We recommend you use the default image sclorg image. If you are coming from a deployment using the old postgres image from dockerhub (postgres:13), upgrading from awx-operator version 2.12.2 and below to 2.15.0+ will handle migrating your data to the new postgresql image (postgresql-15-c9s).
+
+You can no longer configure a custom `postgres_data_path` because it is hardcoded in the quay.io/sclorg/postgresql-15-c9s image.
+
+If you override the postgres image to use a custom postgres image like postgres:15 for example, the default data directory path may be different. These images cannot be used interchangeably.
+
+#### Initialize Postgres data volume
+
+When using a hostPath backed PVC and some other storage classes like longhorn storagfe, the postgres data directory needs to be accessible by the user in the postgres pod (UID 26).
+
+To initialize this directory with the correct permissions, configure the following setting, which will use an init container to set the permissions in the postgres volume.
+
+```yaml
+spec:
+  postgres_data_volume_init: true
+```
+
+Should you need to modify the init container commands, there is an example below.
+
+```yaml
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+```

docs/user-guide/network-and-tls-configuration.md

@@ -32,7 +32,8 @@ The following variables are customizable only when `service_type=LoadBalancer`
 | --------------------- | ---------------------------------------- | ------- |
 | loadbalancer_protocol | Protocol to use for Loadbalancer ingress | http    |
 | loadbalancer_port     | Port used for Loadbalancer ingress       | 80      |
-| loadbalancer_ip        | Assign Loadbalancer IP                   | ''      |  
+| loadbalancer_ip       | Assign Loadbalancer IP                   | ''      |
+| loadbalancer_class    | LoadBalancer class to use                | ''      |
 
 ```yaml
 ---
@@ -42,6 +43,7 @@ spec:
   loadbalancer_ip: '192.168.10.25'
   loadbalancer_protocol: https
   loadbalancer_port: 443
+  loadbalancer_class: service.k8s.aws/nlb
   service_annotations: |
     environment: testing
   service_labels: |
@@ -86,22 +88,26 @@ spec:
 
 The following variables are customizable when `ingress_type=ingress`. The `ingress` type creates an Ingress resource as [documented](https://kubernetes.io/docs/concepts/services-networking/ingress/) which can be shared with many other Ingress Controllers as [listed](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/).
 
-| Name                | Description                              | Default                     |
-| ------------------- | ---------------------------------------- | --------------------------- |
-| ingress_annotations | Ingress annotations                      | Empty string                |
-| ingress_tls_secret  | Secret that contains the TLS information | Empty string                |
-| ingress_class_name  | Define the ingress class name            | Cluster default             |
-| hostname            | Define the FQDN                          | {{ meta.name }}.example.com |
-| ingress_path        | Define the ingress path to the service   | /                           |
-| ingress_path_type   | Define the type of the path (for LBs)    | Prefix                      |
-| ingress_api_version | Define the Ingress resource apiVersion   | 'networking.k8s.io/v1'      |
+| Name                               | Description                                                                        | Default                     |
+| ---------------------------------- | ---------------------------------------------------------------------------------- | --------------------------- |
+| ingress_annotations                | Ingress annotations                                                                | Empty string                |
+| ingress_tls_secret _(deprecated)_  | Secret that contains the TLS information                                           | Empty string                |
+| ingress_class_name                 | Define the ingress class name                                                      | Cluster default             |
+| hostname _(deprecated)_            | Define the FQDN                                                                    | {{ meta.name }}.example.com |
+| ingress_hosts                      | Define one or multiple FQDN with optional Secret that contains the TLS information | Empty string                |
+| ingress_path                       | Define the ingress path to the service                                             | /                           |
+| ingress_path_type                  | Define the type of the path (for LBs)                                              | Prefix                      |
+| ingress_api_version                | Define the Ingress resource apiVersion                                             | 'networking.k8s.io/v1'      |
 
 ```yaml
 ---
 spec:
   ...
   ingress_type: ingress
-  hostname: awx-demo.example.com
+  ingress_hosts:
+    - hostname: awx-demo.example.com
+    - hostname: awx-demo.sample.com
+      tls_secret: sample-tls-secret
   ingress_annotations: |
     environment: testing
 ```
@@ -119,7 +125,10 @@ Some Ingress Controllers need a special configuration to fully support AWX, add
 spec:
   ...
   ingress_type: ingress
-  hostname: awx-demo.example.com
+  ingress_hosts:
+    - hostname: awx-demo.example.com
+    - hostname: awx-demo.sample.com
+      tls_secret: sample-tls-secret
   ingress_controller: contour
 ```
 

hack/publish-to-operator-hub.sh

@@ -15,43 +15,42 @@
 #
 # Usage:
 #   First, check out awx-operator tag you intend to release, in this case, 1.0.0
-#   $ VERSION=1.1.2 PREV_VERSION=1.1.1 FORK=<your-fork> ./publish-operator.sh
+#   $ VERSION=1.1.2 PREV_VERSION=1.1.1 FORK=<your-fork> ./hack/publish-to-operator-hub.sh
 #
 # Remember to change update the VERSION and PREV_VERSION before running!!!
 
-
 set -e
 
-VERSION=${VERSION:-blah2}
-PREV_VERSION=${PREV_VERSION:-blah1}
+VERSION=${VERSION:-$(make print-VERSION)}
+PREV_VERSION=${PREV_VERSION:-$(make print-PREV_VERSION)}
+
 BRANCH=publish-awx-operator-$VERSION
-FORK=${FORK:-fork}
+FORK=${FORK:-awx-auto}
+GITHUB_TOKEN=${GITHUB_TOKEN:-$AWX_AUTO_GITHUB_TOKEN}
 
-IMG=quay.io/ansible/awx-operator:$VERSION
-CATALOG_IMG=quay.io/ansible/awx-operator-catalog:$VERSION
-BUNDLE_IMG=quay.io/ansible/awx-operator-bundle:$VERSION
+IMG_REPOSITORY=${IMG_REPOSITORY:-quay.io/ansible}
 
-# Set path variables
-OPERATOR_PATH=${OPERATOR_PATH:-~/awx-operator}
+OPERATOR_IMG=$IMG_REPOSITORY/awx-operator:$VERSION
+CATALOG_IMG=$IMG_REPOSITORY/awx-operator-catalog:$VERSION
+BUNDLE_IMG=$IMG_REPOSITORY/awx-operator-bundle:$VERSION
 
-# Build & Push Operator Image  # Not needed because it is done as part of the GHA release automation
-# make docker-build docker-push IMG=$IMG
+COMMUNITY_OPERATOR_GITHUB_ORG=${COMMUNITY_OPERATOR_GITHUB_ORG:-k8s-operatorhub}
+COMMUNITY_OPERATOR_PROD_GITHUB_ORG=${COMMUNITY_OPERATOR_PROD_GITHUB_ORG:-redhat-openshift-ecosystem}
 
 # Build bundle directory
-rm -rf bundle/
-make bundle IMG=$IMG
+make bundle IMG=$OPERATOR_IMG
 
 # Build bundle and catalog images
-make bundle-build bundle-push BUNDLE_IMG=$BUNDLE_IMG IMG=$IMG
-make catalog-build catalog-push CATALOG_IMG=$CATALOG_IMG BUNDLE_IMGS=$BUNDLE_IMG BUNDLE_IMG=$BUNDLE_IMG IMG=$IMG
+make bundle-build bundle-push BUNDLE_IMG=$BUNDLE_IMG IMG=$OPERATOR_IMG
+make catalog-build catalog-push CATALOG_IMG=$CATALOG_IMG BUNDLE_IMGS=$BUNDLE_IMG BUNDLE_IMG=$BUNDLE_IMG IMG=$OPERATOR_IMG
 
 # Set containerImage & namespace variables in CSV
-sed -i -e "s|containerImage: quay.io/ansible/awx-operator:devel|containerImage: quay.io/ansible/awx-operator:${VERSION}|g" bundle/manifests/awx-operator.clusterserviceversion.yaml
-sed -i -e "s|namespace: placeholder|namespace: awx|g" bundle/manifests/awx-operator.clusterserviceversion.yaml
+sed -i.bak -e "s|containerImage: quay.io/ansible/awx-operator:devel|containerImage: ${OPERATOR_IMG}|g" bundle/manifests/awx-operator.clusterserviceversion.yaml
+sed -i.bak -e "s|namespace: placeholder|namespace: awx|g" bundle/manifests/awx-operator.clusterserviceversion.yaml
 
 # Add replaces to dependency graph for upgrade path
 if ! grep -qF 'replaces: awx-operator.v${PREV_VERSION}' bundle/manifests/awx-operator.clusterserviceversion.yaml; then
-  sed -i -e "/version: ${VERSION}/a \\
+  sed -i.bak -e "/version: ${VERSION}/a \\
   replaces: awx-operator.v$PREV_VERSION" bundle/manifests/awx-operator.clusterserviceversion.yaml
 fi
 
@@ -60,21 +59,20 @@ mv bundle/manifests/awx-operator.clusterserviceversion.yaml bundle/manifests/awx
 
 # Set Openshift Support Range (bump minKubeVersion in CSV when changing)
 if ! grep -qF 'openshift.versions' bundle/metadata/annotations.yaml; then
-  sed -i -e "/annotations:/a \\
-  com.redhat.openshift.versions: v4.10-v4.13\n" bundle/metadata/annotations.yaml
+  sed -i.bak -e "/annotations:/a \\
+  com.redhat.openshift.versions: v4.11" bundle/metadata/annotations.yaml
 fi
 
-# -- Put up community-operators PR
-cd $OPERATOR_PATH
-git clone git@github.com:k8s-operatorhub/community-operators.git
+# Remove .bak files from bundle result from sed commands
+find bundle -name "*.bak" -type f -delete
+
+echo "-- Create branch on community-operators fork --"
+git clone https://github.com/$COMMUNITY_OPERATOR_GITHUB_ORG/community-operators.git
 
 mkdir -p community-operators/operators/awx-operator/$VERSION/
 cp -r bundle/* community-operators/operators/awx-operator/$VERSION/
-cd community-operators/operators/awx-operator/$VERSION/
-pwd
-ls -la
+pushd community-operators/operators/awx-operator/$VERSION/
 
-# Commit and push PR
 git checkout -b $BRANCH
 git add ./
 git status
@@ -83,26 +81,26 @@ message='operator [N] [CI] awx-operator'
 commitMessage="${message} ${VERSION}"
 git commit -m "$commitMessage" -s
 
-git remote add upstream git@github.com:$FORK/community-operators.git
+git remote add upstream https://$GITHUB_TOKEN@github.com/$FORK/community-operators.git
+
+git push upstream --delete $BRANCH || true
 git push upstream $BRANCH
 
+gh pr create \
+  --title "operator awx-operator (${VERSION})" \
+  --body "operator awx-operator (${VERSION})" \
+  --base main \
+  --head $FORK:$BRANCH \
+  --repo $COMMUNITY_OPERATOR_GITHUB_ORG/community-operators
+popd
 
-# -- Put up community-operators-prod PR
-# Reset directory
-cd $OPERATOR_PATH
-
-pwd
-
-git clone git@github.com:redhat-openshift-ecosystem/community-operators-prod.git
+echo "-- Create branch on community-operators-prod fork --"
+git clone https://github.com/$COMMUNITY_OPERATOR_PROD_GITHUB_ORG/community-operators-prod.git
 
 mkdir -p community-operators-prod/operators/awx-operator/$VERSION/
 cp -r bundle/* community-operators-prod/operators/awx-operator/$VERSION/
-cd community-operators-prod/operators/awx-operator/$VERSION/
+pushd community-operators-prod/operators/awx-operator/$VERSION/
 
-pwd
-ls -la
-
-# Commit and push PR
 git checkout -b $BRANCH
 git add ./
 git status
@@ -111,15 +109,15 @@ message='operator [N] [CI] awx-operator'
 commitMessage="${message} ${VERSION}"
 git commit -m "$commitMessage" -s
 
-git remote add upstream git@github.com:$FORK/community-operators-prod.git
+git remote add upstream https://$GITHUB_TOKEN@github.com/$FORK/community-operators-prod.git
+
+git push upstream --delete $BRANCH || true
 git push upstream $BRANCH
 
-
-# -- Print Links to Branches
-echo "Commnity Operators: https://github.com/$FORK/community-operators/pull/new/$BRANCH"
-echo "Commnity Operators Prod: https://github.com/$FORK/community-operators-prod/pull/new/$BRANCH"
-
-# -- Cleanup
-
-rm -rf $OPERATOR_PATH/community-operators
-rm -rf $OPERATOR_PATH/community-operators-prod
+gh pr create \
+  --title "operator awx-operator (${VERSION})" \
+  --body "operator awx-operator (${VERSION})" \
+  --base main \
+  --head $FORK:$BRANCH \
+  --repo $COMMUNITY_OPERATOR_PROD_GITHUB_ORG/community-operators-prod
+popd

mkdocs.yml

@@ -18,6 +18,9 @@ theme:
     - navigation.indexes
     - navigation.tracking
     - toc.integrate
+    - search.highlight
+    - search.share
+    - search.suggest
   palette:
     - media: "(prefers-color-scheme: light)"
       primary: teal
@@ -63,9 +66,11 @@ nav:
       - user-guide/advanced-configuration/privileged-tasks.md
       - user-guide/advanced-configuration/containers-resource-requirements.md
       - user-guide/advanced-configuration/priority-classes.md
+      - user-guide/advanced-configuration/adding-execution-nodes.md
       - user-guide/advanced-configuration/scaling-the-web-and-task-pods-independently.md
       - user-guide/advanced-configuration/assigning-awx-pods-to-specific-nodes.md
       - user-guide/advanced-configuration/trusting-a-custom-certificate-authority.md
+      - user-guide/advanced-configuration/custom-receptor-certs.md
       - user-guide/advanced-configuration/enabling-ldap-integration-at-awx-bootstrap.md
       - user-guide/advanced-configuration/persisting-projects-directory.md
       - user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md
@@ -80,6 +85,7 @@ nav:
       - user-guide/advanced-configuration/labeling-operator-managed-objects.md
       - user-guide/advanced-configuration/pods-termination-grace-period.md
       - user-guide/advanced-configuration/disable-ipv6.md
+      - user-guide/advanced-configuration/mesh-ingress.md
   - Troubleshooting:
       - troubleshooting/debugging.md
 
@@ -89,7 +95,8 @@ exclude_docs:
 plugins:
   - autorefs
   - markdown-exec
-  - search
+  - material/search:
+      separator: '[\s\-,:!=\[\]()"`/]+|\.(?!\d)|&[lg]t;|(?!\b)(?=[A-Z][a-z])'
   - mkdocstrings:
       handlers:
         python:

molecule/default/molecule.yml

@@ -21,6 +21,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         default_awx_version: "{{ lookup('url', 'https://api.github.com/repos/ansible/awx/releases/latest') | from_json | json_query('tag_name') }}"
@@ -30,6 +31,8 @@ provisioner:
         operator_image: ${OPERATOR_IMAGE:-""}
         operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"}
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"}
 verifier:

molecule/default/tasks/apply_awx_spec.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create or update the awx.ansible.com/v1alpha1.AWX
+- name: Create or update the awx.ansible.com/v1beta1.AWX
   k8s:
     state: present
     namespace: '{{ namespace }}'

molecule/default/tasks/awx_replicas_test.yml

@@ -49,16 +49,16 @@
 ####
 
     - debug:
-        msg: test - replicas=3 should give 3 of each
+        msg: test - replicas=2 should give 2 of each
 
     - include_tasks: apply_awx_spec.yml
       vars:
         additional_fields:
-          replicas: 3
+          replicas: 2
 
     - include_tasks: _test_case_replicas.yml
       vars:
-        expected_web_replicas: 3
-        expected_task_replicas: 3
+        expected_web_replicas: 2
+        expected_task_replicas: 2
   tags:
     - replicas

molecule/default/tasks/awx_test.yml

@@ -1,98 +1,8 @@
 ---
 - include_tasks: apply_awx_spec.yml
 
-- name: Obtain generated admin password
-  k8s_info:
-    namespace: '{{ namespace }}'
-    kind: Secret
-    name: example-awx-admin-password
-  register: admin_pw_secret
-
-- block:
-    - name: Get web pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-web
-      register: awx_web_pod
-      when: not awx_version
-
-    - name: Get task pod details
-      k8s_info:
-        namespace: '{{ namespace }}'
-        kind: Pod
-        label_selectors:
-          - app.kubernetes.io/name = example-awx-task
-      register: awx_task_pod
-      when: not awx_version
-
-    - name: Extract tags from images from web pod
-      set_fact:
-        web_image_tags: |
-          {{ awx_web_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - name: Extract tags from images from task pod
-      set_fact:
-        task_image_tags: |
-          {{ awx_task_pod.resources[0].spec.containers |
-             map(attribute='image') |
-             map('regex_search', default_awx_version) }}
-      when: not awx_version
-
-    - fail:
-        msg: |
-          It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
-          This is an environment variable that is set via build arg when releasing awx-operator.
-      when:
-        - not awx_version
-        - default_awx_version not in web_image_tags
-        - default_awx_version not in task_image_tags
-
-    - name: Launch Demo Job Template
-      awx.awx.job_launch:
-        name: Demo Job Template
-        wait: yes
-        validate_certs: no
-        controller_host: localhost/awx/
-        controller_username: admin
-        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-  rescue:
-    - name: Get list of project updates and jobs
-      uri:
-        url: "http://localhost/awx/api/v2/{{ resource }}/"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      register: job_lists
-      loop:
-        - project_updates
-        - jobs
-      loop_control:
-        loop_var: resource
-
-    - name: Get all job and project details
-      uri:
-        url: "http://localhost{{ endpoint }}"
-        user: admin
-        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
-        force_basic_auth: yes
-      loop: |
-        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
-      loop_control:
-        loop_var: endpoint
-
-    - name: Re-emit failure
-      vars:
-        failed_task:
-          result: '{{ ansible_failed_result }}'
-      fail:
-        msg: '{{ failed_task }}'
-
-- block:
+- name: Validate AWX deployment
+  block:
     - name: Look up details for this AWX instance
       k8s_info:
         namespace: "{{ namespace }}"
@@ -117,65 +27,90 @@
           - app.kubernetes.io/name = example-awx-task
       register: awx_task_pod
 
-    - name: Extract additional_labels from AWX spec
-      set_fact:
-        awx_additional_labels: >-
-          {{ this_awx.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
+    - name: Validate DEFAULT_AWX_VERSION
+      block:
+      - name: Extract tags from images from web pod
+        set_fact:
+          web_image_tags: |
+            {{ awx_web_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - name: Extract tags from images from task pod
+        set_fact:
+          task_image_tags: |
+            {{ awx_task_pod.resources[0].spec.containers |
+              map(attribute='image') |
+              map('regex_search', default_awx_version) }}
+      - fail:
+          msg: |
+            It looks like you may have broken the DEFAULT_AWX_VERSION functionality.
+            This is an environment variable that is set via build arg when releasing awx-operator.
+        when:
+          - default_awx_version not in web_image_tags
+          - default_awx_version not in task_image_tags
+      when: not awx_version
 
-    - name: Extract additional_labels from AWX web Pod
-      set_fact:
-        awx_web_pod_additional_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
-             | list
-          }}
-
-    - name: Extract additional_labels from AWX task Pod
-      set_fact:
-        awx_task_pod_additional_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
+    - name: Validate additional_labels
+      block:
+      - name: Extract additional_labels from AWX spec
+        set_fact:
+          awx_additional_labels: >-
+            {{ this_awx.resources[0].metadata.labels
               | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
               | list
-          }}
+            }}
 
-    - name: Assert AWX web Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX web Pod
+        set_fact:
+          awx_web_pod_additional_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+              | list
+            }}
 
-    - name: Assert AWX task Pod contains additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_additional_labels == awx_additional_labels
+      - name: Extract additional_labels from AWX task Pod
+        set_fact:
+          awx_task_pod_additional_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+                | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels)
+                | list
+            }}
 
-    - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_web_pod_extra_labels: >-
-          {{ awx_web_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Assert AWX web Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_additional_labels == awx_additional_labels
 
-    - name: AWX web Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_web_pod_extra_labels == []
+      - name: Assert AWX task Pod contains additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_additional_labels == awx_additional_labels
 
-    - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
-      set_fact:
-        awx_task_pod_extra_labels: >-
-          {{ awx_task_pod.resources[0].metadata.labels
-             | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
-             | list
-          }}
+      - name: Extract web Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_web_pod_extra_labels: >-
+            {{ awx_web_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
 
-    - name: AWX task Pod doesn't contain AWX labels not in additional_labels
-      ansible.builtin.assert:
-        that:
-          - awx_task_pod_extra_labels == []
+      - name: AWX web Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_web_pod_extra_labels == []
+
+      - name: Extract task Pod labels which shouldn't have been propagated to it from AWX
+        set_fact:
+          awx_task_pod_extra_labels: >-
+            {{ awx_task_pod.resources[0].metadata.labels
+              | dict2items | selectattr('key', 'in', ["my/do-not-inherit"])
+              | list
+            }}
+
+      - name: AWX task Pod doesn't contain AWX labels not in additional_labels
+        ansible.builtin.assert:
+          that:
+            - awx_task_pod_extra_labels == []
 
   rescue:
     - name: Re-emit failure
@@ -184,3 +119,113 @@
           result: '{{ ansible_failed_result }}'
       fail:
         msg: '{{ failed_task }}'
+
+- name: Obtain generated admin password
+  k8s_info:
+    namespace: '{{ namespace }}'
+    kind: Secret
+    name: example-awx-admin-password
+  register: admin_pw_secret
+
+- name: Validate demo job launch
+  block:
+    - name: Launch Demo Job Template
+      awx.awx.job_launch:
+        name: Demo Job Template
+        wait: yes
+        validate_certs: no
+        controller_host: localhost/awx/
+        controller_username: admin
+        controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+
+  rescue:
+    - name: Create debug output directory
+      ansible.builtin.file:
+        path: '{{ debug_output_dir }}'
+        state: directory
+
+    - name: Get list of project updates and jobs
+      uri:
+        url: "http://localhost/awx/api/v2/{{ resource }}/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: job_lists
+      loop:
+        - project_updates
+        - jobs
+      loop_control:
+        loop_var: resource
+
+    - name: Store job_lists debug output
+      copy:
+        content: "{{ job_lists | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_lists.json"
+      when: store_debug_output | default(false)
+
+    - name: Get all job and project_update details
+      uri:
+        url: "http://localhost{{ endpoint }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
+      loop_control:
+        loop_var: endpoint
+      register: job_details
+
+    - name: Store job_details debug output
+      copy:
+        content: "{{ job_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/job_details.json"
+      when: store_debug_output | default(false)
+
+    - name: Get list of instances
+      uri:
+        url: "http://localhost/awx/api/v2/instances/"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      register: instances_list
+
+    - name: Store instances_list debug output
+      copy:
+        content: "{{ instances_list | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_list.json"
+      when: store_debug_output | default(false)
+
+    - name: Get instances detail
+      uri:
+        url: "http://localhost{{ item }}"
+        user: admin
+        password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}"
+        force_basic_auth: yes
+      loop: |
+        {{ instances_list.json.results | map(attribute='url') }}
+      loop_control:
+        loop_var: item
+      register: instances_details
+
+    - name: Store instances_details debug output
+      copy:
+        content: "{{ instances_details | to_nice_json }}"
+        dest: "{{ debug_output_dir }}/instances_details.json"
+      when: store_debug_output | default(false)
+
+    ## TODO: figure out why this doesn't work
+    # - name: Store debug outputs
+    #   copy:
+    #     content: '{{ item }}'
+    #     dest: "{{ debug_output_dir }}/{{ item }}.json"
+    #   loop:
+    #     - job_lists
+    #     - job_details
+    #   when: store_debug_output | default(false)
+
+    - name: Re-emit failure
+      vars:
+        failed_task:
+          result: '{{ ansible_failed_result }}'
+      fail:
+        msg: '{{ failed_task }}'

molecule/default/tasks/awxmeshingress_test.yml

@@ -0,0 +1,19 @@
+# TODO: Add tests for AWXMeshIngress
+# ---
+# - name: Create the awx.ansible.com/v1alpha1.AWXMeshIngress
+#   k8s:
+#     state: present
+#     namespace: '{{ namespace }}'
+#     definition: "{{ lookup('template', '/'.join([samples_dir, cr_file])) | from_yaml }}"
+#     wait: yes
+#     wait_timeout: 300
+#     wait_condition:
+#       type: Successful
+#       status: "True"
+#   vars:
+#     cr_file: 'awx_v1alpha1_awxmeshingress.yaml'
+
+# - name: Add assertions here
+#   assert:
+#     that: false
+#     fail_msg: FIXME Add real assertions for your operator

molecule/default/templates/awx_cr_molecule.yml.j2

@@ -13,6 +13,12 @@ spec:
 {% endif %}
 {% if awx_version %}
   image_version: {{ awx_version }}
+{% endif %}
+{% if awx_ee_image %}
+  control_plane_ee_image: {{ awx_ee_image }}
+  ee_images:
+  - image: {{ awx_ee_image }}
+    name: AWX EE
 {% endif %}
   ingress_type: ingress
   ingress_path: /awx
@@ -32,11 +38,13 @@ spec:
       memory: 16M
   no_log: false
   postgres_resource_requirements: {}
-  postgres_init_container_resource_requirements: {}
   redis_resource_requirements: {}
   additional_labels:
   - my/team
   - my/service
+  extra_settings:
+  - setting: LOG_AGGREGATOR_LEVEL
+    value: "'DEBUG'"
 {% if additional_fields is defined %}
   {{ additional_fields | to_nice_yaml | indent(2) }}
-{% endif %}
+{% endif %}

molecule/default/utils/output_all_container_logs_for_pod.yml

@@ -0,0 +1,15 @@
+---
+- name: Get all container log in pod
+  kubernetes.core.k8s_log:
+    namespace: '{{ namespace }}'
+    name: '{{ item.metadata.name }}'
+    all_containers: true
+  register: all_container_logs
+
+- name: Store logs in file
+  ansible.builtin.copy:
+    content: "{{ all_container_logs.log_lines | join('\n') }}"
+    dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log'
+
+# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs
+# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported

molecule/default/utils/output_k8s_resources.yml

@@ -0,0 +1,29 @@
+---
+- name: Retrieve relevant k8s resources
+  kubernetes.core.k8s_info:
+    api_version: '{{ item.api_version }}'
+    kind: '{{ item.kind }}'
+    namespace: '{{ namespace }}'
+  loop:
+    - api_version: v1
+      kind: Pod
+    - api_version: apps/v1
+      kind: Deployment
+    - api_version: v1
+      kind: Secret
+    - api_version: v1
+      kind: ConfigMap
+    - api_version: "awx.ansible.com/v1beta1"
+      kind: AWX
+  register: debug_resources
+
+- name: debug print item.kind and item.metadata.name
+  debug:
+    msg: '{{ item.kind }}-{{ item.metadata.name }}'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"
+
+- name: Output gathered resource to files
+  ansible.builtin.copy:
+    content: '{{ item | to_nice_json }}'
+    dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json'
+  loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}"

molecule/default/verify.yml

@@ -10,52 +10,41 @@
     ctrl_label: control-plane=controller-manager
 
   tasks:
-    - block:
+    - name: Perform awx tests
+      block:
         - name: Import all test files from tasks/
-          include_tasks: '{{ item }}'
+          ansible.builtin.include_tasks: '{{ item }}'
           with_fileglob:
-            - tasks/*_test.yml
+            - tasks/awx_test.yml
+            - tasks/awx_replicas_test.yml
           tags:
             - always
       rescue:
-        - name: Retrieve relevant resources
-          k8s_info:
-            api_version: '{{ item.api_version }}'
-            kind: '{{ item.kind }}'
+        - name: Create debug output directory
+          ansible.builtin.file:
+            path: '{{ debug_output_dir }}'
+            state: directory
+          tags:
+            - always
+
+        - name: Gather and output K8s resources
+          ansible.builtin.include_tasks: utils/output_k8s_resources.yml
+          tags:
+            - always
+
+        - name: Get all pods
+          kubernetes.core.k8s_info:
+            api_version: v1
+            kind: Pod
             namespace: '{{ namespace }}'
-          loop:
-            - api_version: v1
-              kind: Pod
-            - api_version: apps/v1
-              kind: Deployment
-            - api_version: v1
-              kind: Secret
-            - api_version: v1
-              kind: ConfigMap
-          register: debug_resources
+          register: all_pods
           tags:
             - always
 
-        - name: Retrieve Pod logs
-          k8s_log:
-            name: '{{ item.metadata.name }}'
-            namespace: '{{ namespace }}'
-            container: awx-manager
-          loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}"
-          register: debug_logs
-          tags:
-            - always
-
-        - name: Output gathered resources
-          debug:
-            var: debug_resources
-          tags:
-            - always
-
-        - name: Output gathered logs
-          debug:
-            var: item.log_lines
-          loop: '{{ debug_logs.results }}'
+        - name: Get all container logs for all pods
+          ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml
+          loop: '{{ all_pods.resources }}'
+          ignore_errors: yes
           tags:
             - always
 
@@ -63,7 +52,7 @@
           vars:
             failed_task:
               result: '{{ ansible_failed_result }}'
-          fail:
+          ansible.builtin.fail:
             msg: '{{ failed_task }}'
           tags:
             - always

molecule/kind/molecule.yml

@@ -23,6 +23,7 @@ provisioner:
         namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test}
     host_vars:
       localhost:
+        awx_ee_image: ${AWX_EE_TEST_IMAGE:-""}
         awx_image: ${AWX_TEST_IMAGE:-""}
         awx_version: ${AWX_TEST_VERSION:-""}
         ansible_python_interpreter: '{{ ansible_playbook_python }}'
@@ -34,6 +35,8 @@ provisioner:
         operator_pull_policy: "Never"
         kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
         kustomize: ${KUSTOMIZE_PATH:-kustomize}
+        store_debug_output: ${STORE_DEBUG_OUTPUT:-false}
+        debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"}
   env:
     K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig
     KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig

molecule/requirements.yml

@@ -2,8 +2,6 @@
 collections:
   - name: community.general
   - name: kubernetes.core
-    version: 2.3.2
   - name: operator_sdk.util
   - name: community.docker
-    version: 3.4.5
   - name: awx.awx

noxfile.py

@@ -0,0 +1,20 @@
+import nox
+
+
+@nox.session
+def build(session: nox.Session):
+    """
+    Build the AWX Operator docsite.
+    """
+    session.install(
+        "-r",
+        "docs/requirements.in",
+        "-c",
+        "docs/requirements.txt",
+    )
+    session.run(
+        "mkdocs",
+        "build",
+        "--strict",
+        *session.posargs,
+    )

requirements.yml

@@ -3,4 +3,4 @@ collections:
   - name: kubernetes.core
     version: '>=2.3.2'
   - name: operator_sdk.util
-    version: "0.4.0"
+    version: "0.5.0"

roles/backup/tasks/awx-cro.yml

@@ -31,5 +31,6 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir  }}/awx_object"
     content: "{{ awx_spec | to_yaml }}"

roles/backup/tasks/delete_backup.yml

@@ -3,5 +3,6 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       bash -c 'rm -rf {{ backup_dir  }}'

roles/backup/tasks/dump_ingress_tls_secrets.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Get secret
+  k8s_info:
+    version: v1
+    kind: Secret
+    namespace: '{{ ansible_operator_meta.namespace }}'
+    name: "{{ item }}"
+  register: _secret
+  no_log: "{{ no_log }}"
+
+- name: Backup secret if exists
+  block:
+    - name: Set secret key
+      set_fact:
+        _data: "{{ _secret['resources'][0]['data'] }}"
+        _type: "{{ _secret['resources'][0]['type'] }}"
+      no_log: "{{ no_log }}"
+
+    - name: Create and Add secret names and data to dictionary
+      set_fact:
+        secret_dict: "{{ secret_dict | default({}) | combine({item: { 'name': item, 'data': _data, 'type': _type }}) }}"
+      no_log: "{{ no_log }}"
+  when: _secret | length

roles/backup/tasks/init.yml

@@ -77,10 +77,10 @@
   set_fact:
     _postgres_image: "{{ _custom_postgres_image | default(lookup('env', 'RELATED_IMAGE_AWX_POSTGRES')) | default(_default_postgres_image, true) }}"
 
-- name: Create management pod from templated deployment config
+- name: Create management pod from the template
   k8s:
     name: "{{ ansible_operator_meta.name }}-db-management"
-    kind: Deployment
+    kind: Pod
     state: present
     definition: "{{ lookup('template', 'management-pod.yml.j2') }}"
     wait: true

roles/backup/tasks/postgres.yml

@@ -60,6 +60,7 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       mkdir -p {{ backup_dir }}
 
@@ -67,19 +68,13 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: >-
       touch {{ backup_dir }}/tower.db
 
-- name: Set permissions on file for database dump
-  k8s_exec:
-    namespace: "{{ backup_pvc_namespace }}"
-    pod: "{{ ansible_operator_meta.name }}-db-management"
-    command: >-
-      bash -c "chmod 660 {{ backup_dir }}/tower.db && chown :root {{ backup_dir }}/tower.db"
-
 - name: Set full resolvable host name for postgres pod
   set_fact:
-    resolvable_db_host: '{{ (awx_postgres_type == "managed") | ternary(awx_postgres_host + "." + ansible_operator_meta.namespace + ".svc.cluster.local", awx_postgres_host) }}'  # yamllint disable-line rule:line-length
+    resolvable_db_host: '{{ (awx_postgres_type == "managed") | ternary(awx_postgres_host + "." + ansible_operator_meta.namespace + ".svc", awx_postgres_host) }}'  # yamllint disable-line rule:line-length
   no_log: "{{ no_log }}"
 
 - name: Get the current resource task pod information.
@@ -88,7 +83,7 @@
     kind: Pod
     namespace: '{{ ansible_operator_meta.namespace }}'
     label_selectors:
-      - "app.kubernetes.io/name={{ ansible_operator_meta.name }}-task"
+      - "app.kubernetes.io/name={{ deployment_name }}-task"
       - "app.kubernetes.io/managed-by={{ deployment_type }}-operator"
       - "app.kubernetes.io/component={{ deployment_type }}"
     field_selectors:
@@ -133,12 +128,29 @@
   k8s_exec:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     command: |
-      bash -c """
+      bash -c "
+      function end_keepalive {
+        rc=$?
+        rm -f \"$1\"
+        kill $(cat /proc/$2/task/$2/children 2>/dev/null) 2>/dev/null || true
+        wait $2 || true
+        exit $rc
+      }
+      keepalive_file=\"$(mktemp)\"
+      while [[ -f \"$keepalive_file\" ]]; do
+        echo 'Dumping data from database...'
+        sleep 60
+      done &
+      keepalive_pid=$!
+      trap 'end_keepalive \"$keepalive_file\" \"$keepalive_pid\"' EXIT SIGINT SIGTERM
+      echo keepalive_pid: $keepalive_pid
       set -e -o pipefail
       PGPASSWORD='{{ awx_postgres_pass }}' {{ pgdump }} > {{ backup_dir }}/tower.db
+      set +e +o pipefail
       echo 'Successful'
-      """
+      "
   register: data_migration
   no_log: "{{ no_log }}"
   failed_when: "'Successful' not in data_migration.stdout"

roles/backup/tasks/secrets.yml

@@ -12,11 +12,16 @@
   include_tasks: dump_secret.yml
   loop:
     - route_tls_secret
+    # ingress_tls_secret is deprecated in favor of ingress_hosts.tls_secret
     - ingress_tls_secret
     - ldap_cacert_secret
     - bundle_cacert_secret
     - ee_pull_credentials_secret
 
+- name: Dump ingress tls secret names from awx spec and data into file
+  include_tasks: dump_ingress_tls_secrets.yml
+  with_items: "{{ awx_spec.spec['ingress_hosts'] | default([]) | selectattr('tls_secret', 'defined') | map(attribute='tls_secret') | list }}"
+
 - name: Dump receptor secret names and data into file
   include_tasks: dump_receptor_secrets.yml
   loop:
@@ -45,6 +50,7 @@
   k8s_cp:
     namespace: "{{ backup_pvc_namespace }}"
     pod: "{{ ansible_operator_meta.name }}-db-management"
+    container: "{{ ansible_operator_meta.name }}-db-management"
     remote_path: "{{ backup_dir }}/secrets.yml"
     content: "{{ secrets | to_yaml }}"
   no_log: "{{ no_log }}"

roles/backup/vars/main.yml

@@ -1,8 +1,8 @@
 ---
 deployment_type: "awx"
-_postgres_image: postgres
-_postgres_image_version: 13
+_postgres_image: quay.io/sclorg/postgresql-15-c9s
+_postgres_image_version: latest
 backup_complete: false
 database_type: "unmanaged"
-supported_pg_version: 13
+supported_pg_version: 15
 image_pull_policy: IfNotPresent

roles/installer/defaults/main.yml

@@ -13,6 +13,7 @@ ingress_class_name: ''
 ingress_path: '/'
 ingress_path_type: 'Prefix'
 ingress_api_version: 'networking.k8s.io/v1'
+api_urlpattern_prefix: ''
 # Add annotations to the service account. Specify as literal block. E.g.:
 # service_account_annotations: |
 #   eks.amazonaws.com/role-arn: arn:aws:iam::<ACCOUNT_ID>:role/<IAM_ROLE_NAME>
@@ -40,8 +41,19 @@ ingress_tls_secret: ''
 # ingress_controller: contour
 ingress_controller: ''
 
+# One or multiple FQDN with optional Secret that contains the TLS information.
+# The TLS secret either has to exist before hand with
+# the corresponding cert and key or just be an indicator for where an automated
+# process like cert-manager (enabled via annotations) will store the TLS
+# certificate and key.
+# ingress_hosts:
+#   - hostname: awx-demo.example.com
+#     tls_secret: example-com-tls
+ingress_hosts: ''
+
 loadbalancer_protocol: 'http'
 loadbalancer_port: '80'
+loadbalancer_class: ''
 service_annotations: ''
 
 # Port to be used for NodePort configuration, default is to auto-assign a port between 30000-32768
@@ -233,14 +245,20 @@ ee_pull_credentials_secret: ''
 #     emptyDir: {}
 extra_volumes: ''
 
+# Add extra volumes to the Postgres pod. Specify as literal block. E.g.:
+# postgres_extra_volumes: |
+#   - name: my-volume
+#     emptyDir: {}
+postgres_extra_volumes: ''
+
 # Use these image versions for Ansible AWX.
 
 _image: quay.io/ansible/awx
 _image_version: "{{ lookup('env', 'DEFAULT_AWX_VERSION') or 'latest' }}"
 _redis_image: docker.io/redis
 _redis_image_version: 7
-_postgres_image: postgres
-_postgres_image_version: 13
+_postgres_image: quay.io/sclorg/postgresql-15-c9s
+_postgres_image_version: latest
 image_pull_policy: IfNotPresent
 image_pull_secrets: []
 
@@ -261,10 +279,12 @@ init_container_extra_commands: ''
 init_container_extra_volume_mounts: ''
 
 ee_images:
-  - name: AWX EE (latest)
-    image: quay.io/ansible/awx-ee:latest
+  - name: "AWX EE (latest)"
+    image: "quay.io/ansible/awx-ee:latest"
+  - name: "AWX EE ({{ _image_version }})"
+    image: "quay.io/ansible/awx-ee:{{ _image_version }}"
 
-_control_plane_ee_image: quay.io/ansible/awx-ee:latest
+_control_plane_ee_image: "quay.io/ansible/awx-ee:{{ _image_version }}"
 
 _init_container_image: "{{ _control_plane_ee_image.split(':')[0] }}"
 _init_container_image_version: "{{ _control_plane_ee_image.split(':')[1] }}"
@@ -277,6 +297,11 @@ replicas: 1
 web_replicas: ''
 task_replicas: ''
 
+web_liveness_period: 0
+web_readiness_period: 0
+task_liveness_period: 0
+task_readiness_period: 0
+
 task_args:
   - /usr/bin/launch_awx_task.sh
 task_command: []
@@ -340,13 +365,14 @@ ee_extra_env: ''
 
 # Mount extra volumes on the AWX task/web containers. Specify as literal block.
 # E.g.:
-# task_extra_volume_mounts: ''
+# task_extra_volume_mounts: |
 #   - name: my-volume
 #     mountPath: /some/path
 task_extra_volume_mounts: ''
 web_extra_volume_mounts: ''
 rsyslog_extra_volume_mounts: ''
 ee_extra_volume_mounts: ''
+postgres_extra_volume_mounts: ''
 
 # Add a nodeSelector for the Postgres pods.
 # It must match a node's labels for the pod to be scheduled on that node.
@@ -375,13 +401,9 @@ postgres_resource_requirements:
   requests:
     cpu: 10m
     memory: 64Mi
-postgres_init_container_resource_requirements:
-  requests:
-    cpu: 10m
-    memory: 64Mi
+
 # Assign a preexisting priority class to the postgres pod
 postgres_priority_class: ''
-postgres_data_path: '/var/lib/postgresql/data/pgdata'
 
 # Persistence to the AWX project data folder
 # Whether or not the /var/lib/projects directory will be persistent
@@ -393,6 +415,11 @@ projects_existing_claim: ''
 # Define postgres configuration arguments to use
 postgres_extra_args: ''
 
+postgres_data_volume_init: false
+postgres_init_container_commands: |
+  chown 26:0 /var/lib/pgsql/data
+  chmod 700 /var/lib/pgsql/data
+
 # Configure postgres connection keepalive
 postgres_keepalives: true
 postgres_keepalives_idle: 5
@@ -424,6 +451,7 @@ garbage_collect_secrets: false
 development_mode: false
 
 security_context_settings: {}
+postgres_security_context_settings: {}
 
 # Set no_log settings on certain tasks
 no_log: true
@@ -463,3 +491,5 @@ nginx_worker_processes: 1
 nginx_worker_connections: "{{ uwsgi_listen_queue_size }}"
 nginx_worker_cpu_affinity: 'auto'
 nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}"
+
+extra_settings_files: {}

roles/installer/tasks/database_configuration.yml

@@ -100,20 +100,45 @@
     awx_postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
     awx_postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
     awx_postgres_sslmode: "{{ pg_config['resources'][0]['data']['sslmode'] |  default('prefer'|b64encode) | b64decode }}"
+    awx_postgres_target_session_attrs: "{{ pg_config['resources'][0]['data']['target_session_attrs'] | default('') | b64decode }}"
   no_log: "{{ no_log }}"
 
 - name: Set database as managed
   set_fact:
     managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"
 
-- name: Get the old postgres pod information
+# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades.
+# So we have to take that into account and preferentially set the most recent one.
+- name: Get the old postgres pod (N-1)
   k8s_info:
     kind: Pod
     namespace: "{{ ansible_operator_meta.namespace }}"
-    name: "{{ ansible_operator_meta.name }}-postgres-0"
     field_selectors:
       - status.phase=Running
-  register: old_postgres_pod
+  register: _running_pods
+
+- block:
+    - name: Filter pods by name
+      set_fact:
+        filtered_old_postgres_pods: "{{ _running_pods.resources |
+          selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
+          rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') |
+          list }}"
+
+  # Sort pods by name in reverse order (most recent PG version first) and set
+    - name: Set info for previous postgres pod
+      set_fact:
+        sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
+          sort(attribute='metadata.name') |
+          reverse | list }}"
+      when: filtered_old_postgres_pods | length
+
+
+    - name: Set info for previous postgres pod
+      set_fact:
+        old_postgres_pod: "{{ sorted_old_postgres_pods | first }}"
+      when: filtered_old_postgres_pods | length
+  when: _running_pods.resources | length
 
 - name: Look up details for this deployment
   k8s_info:
@@ -123,30 +148,40 @@
     namespace: "{{ ansible_operator_meta.namespace }}"
   register: this_awx
 
-- name: Check if postgres pod is running and version 12
-  block:
-    - name: Set path to PG_VERSION file for given container image
-      set_fact:
-        path_to_pg_version: '{{ postgres_data_path }}/PG_VERSION'
+# If this deployment has been upgraded before or if upgrade has already been started, set this var
+- name: Set previous PG version var
+  set_fact:
+    _previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
+  when:
+    - "'upgradedPostgresVersion' in this_awx['resources'][0]['status']"
 
+- name: Check if postgres pod is running an older version
+  block:
     - name: Get old PostgreSQL version
       k8s_exec:
         namespace: "{{ ansible_operator_meta.namespace }}"
-        pod: "{{ ansible_operator_meta.name }}-postgres-0"
+        pod: "{{ old_postgres_pod['metadata']['name'] }}"
         command: |
           bash -c """
-          cat {{ path_to_pg_version }}
+          if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then
+            cat "{{ _postgres_data_path }}/PG_VERSION"
+          elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then
+            cat '/var/lib/postgresql/data/pgdata/PG_VERSION'
+          fi
           """
       register: _old_pg_version
 
-    - name: Upgrade data dir from Postgres 12 to 13 if applicable
+    - debug:
+        msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"
+
+    - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
       include_tasks: upgrade_postgres.yml
       when:
-        - _old_pg_version.stdout | default('0') | trim == '12'
+        - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
   when:
     - managed_database
-    - this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default('none') != '12'
-    - old_postgres_pod['resources'] | length  # upgrade is complete and old pg pod has been removed
+    - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version | int < supported_pg_version, true)
+    - old_postgres_pod | length  # If empty, then old pg pod has been removed and we can assume the upgrade is complete
 
 - block:
     - name: Create Database if no database is specified
@@ -167,7 +202,7 @@
       kubernetes.core.k8s_scale:
         api_version: apps/v1
         kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres-13"
+        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
         namespace: "{{ ansible_operator_meta.namespace }}"
         replicas: 0
         wait: yes
@@ -177,7 +212,7 @@
         state: absent
         api_version: apps/v1
         kind: StatefulSet
-        name: "{{ ansible_operator_meta.name }}-postgres-13"
+        name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
         namespace: "{{ ansible_operator_meta.namespace }}"
         wait: yes
       when: create_statefulset_result.error == 422

roles/installer/tasks/enable_metrics_utility.yml

@@ -0,0 +1,42 @@
+---
+- name: Setup PVC if using directory ship target
+  block:
+
+    # Check to make sure provided pvc exists
+    - name: Check provided PVC claim exists
+      kubernetes.core.k8s_info:
+        name: "{{ _metrics_utility_pvc_claim }}"
+        kind: PersistentVolumeClaim
+        namespace: "{{ ansible_operator_meta.namespace }}"
+      when:
+        - _metrics_utility_pvc_claim | length
+
+    - name: Create PVC for metrics-utility
+      kubernetes.core.k8s:
+        kind: PersistentVolumeClaim
+        definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}"
+
+  when: _metrics_utility_ship_target == "directory"
+
+- name: Create default metrics-utility Kubernetes CronJobs
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', item.template) }}"
+    apply: true
+    wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
+  loop:
+    - {name: 'metrics-utility-gather', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+    - {name: 'metrics-utility-report', template: 'cronjobs/metrics-utility-report.yaml.j2'}
+
+- name: Create metrics-utility Kubernetes CronJob for Red Hat Hybrid Cloud Console
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', item.template) }}"
+    apply: true
+    wait: true
+  vars:
+    cronjob_name: "{{ item.name }}"
+    metrics_utility_ship_target: crc  # TODO - Update to console when changed
+  loop:
+    - {name: 'metrics-utility-gather-console', template: 'cronjobs/metrics-utility-gather.yaml.j2'}
+  when: _metrics_utility_console_enabled

roles/installer/tasks/initialize_django.yml

@@ -2,8 +2,8 @@
 - name: Check if there are any super users defined.
   k8s_exec:
     namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
+    pod: "{{ awx_web_pod_name }}"
+    container: "{{ ansible_operator_meta.name }}-web"
     command: >-
       bash -c "echo 'from django.contrib.auth.models import User;
       nsu = User.objects.filter(is_superuser=True, username=\"{{ admin_user }}\").count();
@@ -16,8 +16,8 @@
 - name: Create super user via Django if it doesn't exist.
   k8s_exec:
     namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
+    pod: "{{ awx_web_pod_name }}"
+    container: "{{ ansible_operator_meta.name }}-web"
     command: awx-manage createsuperuser --username={{ admin_user | quote }} --email={{ admin_email | quote }} --noinput
   register: result
   changed_when: "'That username is already taken' not in result.stderr"
@@ -28,8 +28,8 @@
 - name: Update Django super user password
   k8s_exec:
     namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
+    pod: "{{ awx_web_pod_name }}"
+    container: "{{ ansible_operator_meta.name }}-web"
     command: awx-manage update_password --username='{{ admin_user }}' --password='{{ admin_password }}'
   register: result
   changed_when: "'Password updated' in result.stdout"
@@ -39,8 +39,8 @@
 - name: Check if legacy queue is present
   k8s_exec:
     namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
+    pod: "{{ awx_web_pod_name }}"
+    container: "{{ ansible_operator_meta.name }}-web"
     command: >-
       bash -c "awx-manage list_instances | grep '^\[tower capacity=[0-9]*\]'"
   register: legacy_queue
@@ -50,8 +50,8 @@
 - name: Unregister legacy queue
   k8s_exec:
     namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
+    pod: "{{ awx_web_pod_name }}"
+    container: "{{ ansible_operator_meta.name }}-web"
     command: >-
       bash -c "awx-manage unregister_queue --queuename=tower"
   when: "'[tower capacity=' in legacy_queue.stdout"
@@ -74,8 +74,8 @@
 - name: Register default execution environments (without authentication)
   k8s_exec:
     namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
+    pod: "{{ awx_web_pod_name }}"
+    container: "{{ ansible_operator_meta.name }}-web"
     command: >-
       bash -c "awx-manage register_default_execution_environments"
   register: ree
@@ -95,8 +95,8 @@
     - name: Register default execution environments (with authentication)
       k8s_exec:
         namespace: "{{ ansible_operator_meta.namespace }}"
-        pod: "{{ awx_task_pod_name }}"
-        container: "{{ ansible_operator_meta.name }}-task"
+        pod: "{{ awx_web_pod_name }}"
+        container: "{{ ansible_operator_meta.name }}-web"
         command: >-
           bash -c "awx-manage register_default_execution_environments
           --registry-username='{{ default_execution_environment_pull_credentials_user }}'
@@ -111,8 +111,8 @@
 - name: Create preload data if necessary.  # noqa 305
   k8s_exec:
     namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
+    pod: "{{ awx_web_pod_name }}"
+    container: "{{ ansible_operator_meta.name }}-web"
     command: >-
       bash -c "awx-manage create_preload_data"
   register: cdo

roles/installer/tasks/install.yml

@@ -2,7 +2,7 @@
 - name: Delete old deployment for before installing during upgrade
   k8s:
     kind: Deployment
-    api_version: v1
+    api_version: apps/v1
     namespace: "{{ ansible_operator_meta.namespace }}"
     name: "{{ ansible_operator_meta.name }}"
     state: absent
@@ -77,36 +77,34 @@
   - ingress_type | lower == 'route'
   - route_tls_secret != ''
 
+- name: Wait for {{ deployment_type }}restore to complete
+  kubernetes.core.k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ deployment_type }}restore"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: restore_status_check
+  until:
+  # yamllint disable-line rule:line-length
+  - (restore_status_check.resources | length == 0) or (restore_status_check.resources | selectattr('spec.deployment_name', 'equalto', ansible_operator_meta.name) | map(attribute='status') | selectattr('restoreComplete', 'defined') | map(attribute='restoreComplete') | list | length > 0)
+  delay: 10
+  retries: 8640
+  ignore_errors: yes
+  changed_when: false
+
 - name: Include resources configuration tasks
   include_tasks: resources_configuration.yml
 
-- name: Check for pending migrations
-  k8s_exec:
-    namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
-    command: >-
-      bash -c "awx-manage showmigrations | grep -v '[X]' | grep '[ ]' | wc -l"
-  changed_when: false
-  when: awx_task_pod_name != ''
-  register: database_check
-
-- name: Migrate the database if the K8s resources were updated.  # noqa 305
-  k8s_exec:
-    namespace: "{{ ansible_operator_meta.namespace }}"
-    pod: "{{ awx_task_pod_name }}"
-    container: "{{ ansible_operator_meta.name }}-task"
-    command: >-
-      bash -c "awx-manage migrate --noinput"
-  register: migrate_result
-  when:
-  - awx_task_pod_name != ''
-  - database_check is defined
-  - (database_check.stdout|trim) != '0'
+- name: Migrate database to the latest schema
+  include_tasks: migrate_schema.yml
+  when: awx_web_pod_name != ''
 
 - name: Initialize Django
   include_tasks: initialize_django.yml
-  when: awx_task_pod_name != ''
+  when: awx_web_pod_name != ''
+
+- name: Enable optional metrics-utility
+  include_tasks: enable_metrics_utility.yml
+  when: metrics_utility_enabled | bool
 
 - name: Update status variables
   include_tasks: update_status.yml