Merge pull request #1305 from john-westcott-iv/no_jmes

Modifying stage playbook to not need jmespath

.github/workflows/ci.yaml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   molecule:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     name: molecule
     env:
       DOCKER_API_VERSION: "1.38"
@@ -40,7 +40,7 @@ jobs:
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind
   helm:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     name: helm
     steps:
       - uses: actions/checkout@v2

.github/workflows/devel.yaml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   release:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     name: Push devel image
     steps:
       - uses: actions/checkout@v2

.github/workflows/feature.yml

@@ -0,0 +1,75 @@
+---
+
+name: Feature Branch Image Build and Push
+
+on:
+  push:
+    branches: [feature_*]
+
+jobs:
+  release:
+    runs-on: ubuntu-18.04
+    name: Push devel image
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # needed so that git describe --tag works
+
+      - name: Set VERSION
+        run: |
+          echo "VERSION=$(git describe --tags)" >>${GITHUB_ENV}
+
+      - name: Set lower case owner name
+        run: |
+          echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
+        env:
+          OWNER: '${{ github.repository_owner }}'
+
+      - name: Set IMAGE_TAG_BASE
+        run: |
+          echo "IMAGE_TAG_BASE=ghcr.io/${OWNER_LC}/awx-operator" >>${GITHUB_ENV}
+
+      - name: Set ARCH environment variable
+        run: |
+          echo "ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)" >>${GITHUB_ENV}
+
+      - name: Set OS environment variable
+        run: |
+          echo "OS=$(uname | awk '{print tolower($0)}')" >>${GITHUB_ENV}
+
+      - name: Install operator-sdk
+        run: |
+          echo "Installing operator-sdk ${OPERATOR_SDK_DL_URL}" && \
+          curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} && \
+          chmod +x operator-sdk_${OS}_${ARCH} && \
+          sudo mkdir -p /usr/local/bin/ && \
+          sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk && \
+          operator-sdk version
+        env:
+          OPERATOR_SDK_DL_URL: https://github.com/operator-framework/operator-sdk/releases/download/v1.26.0
+
+      - name: Log in to registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Build and Push awx-operator Image
+        run: |
+          make docker-build docker-push
+          docker tag ${IMAGE_TAG_BASE}:${VERSION} ${IMAGE_TAG_BASE}:${GITHUB_REF##*/}
+          docker push ${IMAGE_TAG_BASE}:${GITHUB_REF##*/}
+
+      - name: Build bundle manifests
+        run: |
+          make bundle
+
+      - name: Build and Push awx-operator Bundle
+        run: |
+          make bundle-build bundle-push
+          docker tag ${IMAGE_TAG_BASE}-bundle:v${VERSION} ${IMAGE_TAG_BASE}-bundle:${GITHUB_REF##*/}
+          docker push ${IMAGE_TAG_BASE}-bundle:${GITHUB_REF##*/}
+
+      - name: Build and Push awx-operator Catalog
+        run: |
+          make catalog-build catalog-push
+          docker tag ${IMAGE_TAG_BASE}-catalog:v${VERSION} ${IMAGE_TAG_BASE}-catalog:${GITHUB_REF##*/}
+          docker push ${IMAGE_TAG_BASE}-catalog:${GITHUB_REF##*/}

.github/workflows/promote.yaml

@@ -39,5 +39,5 @@ jobs:
             -e operator_image=quay.io/${{ github.repository }} \
             -e chart_owner=${{ github.repository_owner }} \
             -e tag=${{ github.event.release.tag_name }} \
-            -e gh_token=${{ secrets.GITHUB_TOKEN }}
+            -e gh_token=${{ secrets.GITHUB_TOKEN }} \
             -e gh_user=${{ github.actor }}

README.md

@@ -403,12 +403,14 @@ The following variables are customizable only when `service_type=LoadBalancer`
 | --------------------- | ---------------------------------------- | ------- |
 | loadbalancer_protocol | Protocol to use for Loadbalancer ingress | http    |
 | loadbalancer_port     | Port used for Loadbalancer ingress       | 80      |
+| loadbalancer_ip        | Assign Loadbalancer IP                   | ''      |  
 
 ```yaml
 ---
 spec:
   ...
   service_type: LoadBalancer
+  loadbalancer_ip: '192.168.10.25'
   loadbalancer_protocol: https
   loadbalancer_port: 443
   service_annotations: |
@@ -870,7 +872,7 @@ A sample of extra settings can be found as below. All possible options can be fo
       value: 'LDAPSearch("OU=Groups,DC=abc,DC=com",ldap.SCOPE_SUBTREE,"(objectClass=group)",)'
 
     - setting: AUTH_LDAP_GROUP_TYPE
-      value: 'GroupOfNamesType(name_attr="cn")'
+      value: 'GroupOfNamesType'
 
     - setting: AUTH_LDAP_USER_ATTR_MAP
       value: '{"first_name": "givenName","last_name": "sn","email": "mail"}'

ansible/helm-release.yml

@@ -14,47 +14,78 @@
           Release must exist before running this playbook
       when: release is not success
 
-    - name: Build and package helm chart
-      command: |
-        make helm-package
-      environment:
-        VERSION: "{{ tag }}"
-        IMAGE_TAG_BASE: "{{ operator_image }}"
-      args:
-        chdir: "{{ playbook_dir }}/../"
+    - name: Set helm filename and commit message
+      set_fact:
+        asset_already_attached: False
+        helm_file_name: "awx-operator-{{ tag }}.tgz"
+        commit_message: "Updated index.yaml for release {{ release.json.tag_name }}"
 
-    # Move to chart releaser after https://github.com/helm/chart-releaser/issues/122 exists
-    - name: Upload helm chart
-      uri:
-        url: "https://uploads.github.com/repos/{{ chart_owner }}/{{ chart_repo }}/releases/{{ release.json.id }}/assets?name=awx-operator-{{ tag }}.tgz"
-        src: "{{ playbook_dir }}/../.cr-release-packages/awx-operator-{{ tag }}.tgz"
-        headers:
-          Authorization: "token {{ gh_token }}"
-          Content-Type: "application/octet-stream"
-        status_code:
-          - 200
-          - 201
-      register: asset_upload
-      changed_when: asset_upload.json.state == "uploaded"
+    - name: See if file is already attached
+      set_fact:
+        asset_already_attached: True
+      loop: "{{ release.json.get('assets', []) }}"
+      loop_control:
+        label: "{{ item.name }}"
+      when: item.name == helm_file_name
 
-    - name: Configure git config
-      run: |
-        git config user.name "{{ gh_user }}"
-        git config user.email "{{ gh_user }}@users.noreply.github.com"
+    - when: not asset_already_attached
+      block:
+        - name: Build and package helm chart
+          command: |
+            make helm-package
+          environment:
+            VERSION: "{{ tag }}"
+            IMAGE_TAG_BASE: "{{ operator_image }}"
+          args:
+            chdir: "{{ playbook_dir }}/../"
 
-    - name: Publish helm index
-      command: |
-        make helm-index
-      environment:
-        CHART_OWNER: "{{ chart_owner }}"
-        CR_TOKEN: "{{ gh_token }}"
-      args:
-        chdir: "{{ playbook_dir }}/../"
+        # Move to chart releaser after https://github.com/helm/chart-releaser/issues/122 exists
+        - name: Upload helm chart
+          uri:
+            url: "https://uploads.github.com/repos/{{ chart_owner }}/{{ chart_repo }}/releases/{{ release.json.id }}/assets?name={{ helm_file_name }}"
+            src: "{{ playbook_dir }}/../.cr-release-packages/awx-operator-{{ tag }}.tgz"
+            headers:
+              Authorization: "token {{ gh_token }}"
+              Content-Type: "application/octet-stream"
+            status_code:
+              - 200
+              - 201
+          register: asset_upload
+          changed_when: asset_upload.json.state == "uploaded"
 
-    - name: Stage and Push commit to gh-pages branch
-      command: |
-        git add index.yaml
-        git commit -m "Updated index.yaml latest release"
-        git push
-      args:
+    - name: Ensure gh-pages exists
+      file:
+        state: directory
+        path: "{{ playbook_dir }}/../gh-pages"
+
+    - name: Check if we have published the release
+      command:
+        cmd: "git log --grep='{{ commit_message }}'"
         chdir: "{{ playbook_dir }}/../gh-pages"
+      register: commits_for_release
+
+    - when: commits_for_release.stdout == ''
+      block:
+        - name: Configure git config
+          shell: |
+            git config user.name {{ gh_user }}
+            git config user.email {{ gh_user }}@users.noreply.github.com
+          args:
+            chdir: "{{ playbook_dir }}/../gh-pages"
+
+        - name: Publish helm index
+          command: |
+            make helm-index
+          environment:
+            CHART_OWNER: "{{ chart_owner }}"
+            CR_TOKEN: "{{ gh_token }}"
+          args:
+            chdir: "{{ playbook_dir }}/../"
+
+        - name: Stage and Push commit to gh-pages branch
+          shell: |
+            git add index.yaml
+            git commit -m "{{ commit_message }}"
+            git push
+          args:
+            chdir: "{{ playbook_dir }}/../gh-pages"

config/crd/bases/awx.ansible.com_awxrestores.yaml

@@ -39,12 +39,14 @@ spec:
           spec:
             type: object
             x-kubernetes-preserve-unknown-fields: true
+            required:
+            - deployment_name
             properties:
               backup_source:
                 description: Backup source
                 type: string
                 enum:
-                - CR
+                - Backup CR
                 - PVC
               deployment_name:
                 description: Name of the restored deployment. This should be different from the original deployment name

config/crd/bases/awx.ansible.com_awxs.yaml

@@ -132,6 +132,10 @@ spec:
                 description: Port to use for the loadbalancer
                 type: integer
                 default: 80
+              loadbalancer_ip:
+                description: Assign LoadBalancer IP address
+                type: string
+                default: ''
               route_host:
                 description: The DNS to use to points to the instance
                 type: string

config/manifests/bases/awx-operator.clusterserviceversion.yaml

@@ -270,6 +270,12 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:number
         - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer
+      - displayName: LoadBalancer IP
+        path: loadbalancer_ip
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:string
+        - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer
       - displayName: Route API Version
         path: route_api_version
         x-descriptors:

config/rbac/role.yaml

@@ -20,7 +20,6 @@ rules:
       - watch
   - apiGroups:
       - ""
-      - "rbac.authorization.k8s.io"
     resources:
       - pods
       - services
@@ -31,6 +30,17 @@ rules:
       - events
       - configmaps
       - secrets
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - "rbac.authorization.k8s.io"
+    resources:
       - roles
       - rolebindings
     verbs:
@@ -43,12 +53,22 @@ rules:
       - watch
   - apiGroups:
       - apps
-      - networking.k8s.io
     resources:
       - deployments
       - daemonsets
       - replicasets
       - statefulsets
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
       - ingresses
     verbs:
       - get

docs/debugging.md

@@ -1,4 +1,67 @@
-# Iterating on the installer without deploying the operator
+# Debugging the AWX Operator
+
+## General Debugging
+
+When the operator is deploying AWX, it is running the `installer` role inside the operator container. If the AWX CR's status is `Failed`, it is often useful to look at the awx-operator container logs, which shows the output of the installer role. To see these logs, run:
+
+```
+kubectl logs deployments/awx-operator-controller-manager -c awx-manager -f
+```
+
+### Inspect k8s Resources
+
+Past that, it is often useful to inspect various resources the AWX Operator manages like:
+* awx
+* awxbackup
+* awxrestore
+* pod
+* deployment
+* pvc
+* service
+* ingress
+* route
+* secrets
+* serviceaccount
+
+And if installing via OperatorHub and OLM:
+* subscription
+* csv
+* installPlan
+* catalogSource
+
+To inspect these resources you can use these commands
+
+```
+# Inspecting k8s resources
+kubectl describe -n <namespace> <resource> <resource-name>
+kubectl get -n <namespace> <resource> <resource-name> -o yaml
+kubectl logs -n <namespace> <resource> <resource-name>
+
+# Inspecting Pods
+kubectl exec -it -n <namespace> <pod> <pod-name>
+```
+
+
+### Configure No Log
+
+It is possible to show task output for debugging by setting no_log to false on the AWX CR spec.
+This will show output in the awx-operator logs for any failed tasks where no_log was set to true.
+
+For example:
+
+```
+---
+apiVersion: awx.ansible.com/v1beta1
+kind: AWX
+metadata:
+  name: awx-demo
+spec:
+  service_type: nodeport
+  no_log: false                  # <------------
+
+```
+
+## Iterating on the installer without deploying the operator
 
 Go through the [normal basic install](https://github.com/ansible/awx-operator/blob/devel/README.md#basic-install) steps.
 

docs/migration.md

@@ -34,7 +34,7 @@ metadata:
   namespace: <target namespace>
 stringData:
   host: <external ip or url resolvable by the cluster>
-  port: <external port, this usually defaults to 5432>
+  port: "<external port, this usually defaults to 5432>"    # quotes are required
   database: <desired database name>
   username: <username to connect as>
   password: <password to connect with>

roles/backup/tasks/postgres.yml

@@ -50,7 +50,7 @@
 
 - name: Determine the timestamp for the backup once for all nodes
   set_fact:
-    now: '{{ lookup("pipe", "date +%F-%T") }}'
+    now: '{{ lookup("pipe", "date +%F-%H%M%S") }}'
 
 - name: Set backup directory name
   set_fact:

roles/installer/templates/configmaps/config.yaml.j2

@@ -100,6 +100,7 @@ data:
         include       /etc/nginx/mime.types;
         default_type  application/octet-stream;
         server_tokens off;
+        client_max_body_size 5M;
     
         log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                           '$status $body_bytes_sent "$http_referer" '

roles/installer/templates/deployments/deployment.yaml.j2

@@ -27,7 +27,7 @@ spec:
     "secrets/app_credentials",
     "storage/persistent",
   ] %}
-        checksum-{{ template | replace('/', '-') }}: "{{ lookup('template', template + '.yaml.j2') | md5 }}"
+        checksum-{{ template | replace('/', '-') }}: "{{ lookup('template', template + '.yaml.j2') | sha1 }}"
 {% endfor %}
 {% for secret in [
     "bundle_cacert",
@@ -37,7 +37,7 @@ spec:
     "receptor_ca",
     "receptor_work_signing",
   ] %}
-        checksum-secret-{{ secret }}: "{{ lookup('ansible.builtin.vars', secret, default='')["resources"][0]["data"] | default('') | md5 }}"
+        checksum-secret-{{ secret }}: "{{ lookup('ansible.builtin.vars', secret, default='')["resources"][0]["data"] | default('') | sha1 }}"
 {% endfor %}
 {% if annotations %}
         {{ annotations | indent(width=8) }}

roles/installer/templates/networking/service.yaml.j2

@@ -53,6 +53,9 @@ spec:
   type: NodePort
 {% elif service_type | lower == "loadbalancer" %}
   type: LoadBalancer
+{% if variable is defined and variable|length %}
+  loadbalancerip: '{{ loadbalancer_ip }}'
+{% endif %}
 {% else %}
   type: ClusterIP
 {% endif %}

roles/restore/tasks/import_vars.yml

@@ -2,24 +2,19 @@
 
 - name: Import awx_object variables
   block:
-  - name: Get AWX object definition from pvc
-    k8s_exec:
-      namespace: "{{ backup_pvc_namespace }}"
-      pod: "{{ ansible_operator_meta.name }}-db-management"
-      command: >-
-        bash -c "cat '{{ backup_dir }}/awx_object'"
-    register: awx_object
 
   - name: Create temp file for spec dict
     tempfile:
       state: file
     register: tmp_spec
 
-  - name: Write spec vars to temp file
-    copy:
-      content: "{{ awx_object.stdout }}"
-      dest: "{{ tmp_spec.path }}"
-      mode: '0644'
+  - name: Get AWX object definition from pvc
+    k8s_cp:
+      namespace: "{{ backup_pvc_namespace }}"
+      pod: "{{ ansible_operator_meta.name }}-db-management"
+      remote_path: "{{ backup_dir  }}/awx_object"
+      local_path: "{{ tmp_spec.path }}"
+      state: from_pod
 
   - name: Include spec vars to save them as a dict
     include_vars: "{{ tmp_spec.path }}"

roles/restore/tasks/secrets.yml

@@ -1,25 +1,18 @@
 ---
 
-- name: Get secret definition from pvc
-  k8s_exec:
-    namespace: "{{ backup_pvc_namespace }}"
-    pod: "{{ ansible_operator_meta.name }}-db-management"
-    command: >-
-      bash -c "cat '{{ backup_dir }}/secrets.yml'"
-  register: _secrets
-  no_log: "{{ no_log }}"
-
 - name: Create Temporary secrets file
   tempfile:
     state: file
     suffix: .json
   register: tmp_secrets
 
-- name: Write vars to file locally
-  copy:
-    dest: "{{ tmp_secrets.path }}"
-    content: "{{ _secrets.stdout }}"
-    mode: 0640
+- name: Get secret definition from pvc
+  k8s_cp:
+    namespace: "{{ backup_pvc_namespace }}"
+    pod: "{{ ansible_operator_meta.name }}-db-management"
+    remote_path: "{{ backup_dir  }}/secrets.yml"
+    local_path: "{{ tmp_secrets.path }}"
+    state: from_pod
   no_log: "{{ no_log }}"
 
 - name: Include secret vars from backup