From e37c091d17f8e416c45eef377561629cfccdafa3 Mon Sep 17 00:00:00 2001 From: Marcelo Moreira de Mello Date: Tue, 18 May 2021 14:22:56 -0400 Subject: [PATCH] Make tower_ingress_type to respect ClusterIP definition --- README.md | 266 ++++++++++++------ ansible/group_vars/all | 1 + ansible/instantiate-awx-deployment.yml | 3 +- ansible/templates/crd.yml.j2 | 18 +- ansible/templates/operator.yml.j2 | 2 + deploy/awx-operator.yaml | 24 +- deploy/crds/awx_v1beta1_crd.yaml | 18 +- .../awx-operator.clusterserviceversion.yaml | 27 +- docs/awx-demo.svg | 1 + molecule/test-local/converge.yml | 1 + molecule/test-minikube/converge.yml | 1 + roles/installer/defaults/main.yml | 3 +- roles/installer/templates/ingress.yaml.j2 | 4 +- roles/installer/templates/service.yaml.j2 | 10 +- scripts/build.sh | 3 +- 15 files changed, 256 insertions(+), 126 deletions(-) create mode 100644 docs/awx-demo.svg diff --git a/README.md b/README.md index 3ba71556..72d083df 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w * [Basic Install](#basic-install) * [Admin user account configuration](#admin-user-account-configuration) * [Network and TLS Configuration](#network-and-tls-configuration) + * [Service Type](#service-type) * [Ingress Type](#ingress-type) - * [TLS Termination](#tls-termination) * [Database Configuration](#database-configuration) * [External PostgreSQL Service](#external-postgresql-service) * [Migrating data from an old AWX instance](#migrating-data-from-an-old-awx-instance) @@ -32,8 +32,9 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w * [Upgrading](#upgrading) * [Contributing](#contributing) * [Release Process](#release-process) - * [Build a new release](#build-a-new-release) - * [Build a new version of the operator yaml file](#build-a-new-version-of-the-operator-yaml-file) + * [Verifiy Functionality](#verify-functionality) + * [Update Version](#update-version) + * [Commit / Create Release](#commit--create-release) * [Author](#author) @@ -41,7 +42,7 @@ An [Ansible AWX](https://github.com/ansible/awx) operator for Kubernetes built w This operator is meant to provide a more Kubernetes-native installation method for AWX via an AWX Custom Resource Definition (CRD). -Note that the operator is not supported by Red Hat, and is in **alpha** status. For now, use it at your own risk! +> :warning: The operator is not supported by Red Hat, and is in **alpha** status. For now, use it at your own risk! ## Usage @@ -49,40 +50,119 @@ Note that the operator is not supported by Red Hat, and is in **alpha** status. This Kubernetes Operator is meant to be deployed in your Kubernetes cluster(s) and can manage one or more AWX instances in any namespace. -First, you need to deploy AWX Operator into your cluster. Start by going to https://github.com/ansible/awx-operator/releases and making note of the latest release. - -Replace `` in the URL below with the version you are deploying: +For testing purposes, the `awx-operator` can be deployed on a [Minikube](https://minikube.sigs.k8s.io/docs/) cluster. Due to different OS and hardware environments, please refer to the official Minikube documentation for further information. ```bash -#> kubectl apply -f https://raw.githubusercontent.com/ansible/awx-operator//deploy/awx-operator.yaml +$ minikube start --addons=ingress --cpus=4 --cni=flannel --install-addons=true \ + --kubernetes-version=stable --memory=6g +😄 minikube v1.20.0 on Fedora 34 +✨ Using the kvm2 driver based on user configuration +👍 Starting control plane node minikube in cluster minikube +🔥 Creating kvm2 VM (CPUs=4, Memory=6144MB, Disk=20000MB) ... +🐳 Preparing Kubernetes v1.20.2 on Docker 20.10.6 ... + ▪ Generating certificates and keys ... + ▪ Booting up control plane ... + ▪ Configuring RBAC rules ... +🔗 Configuring Flannel (Container Networking Interface) ... +🔎 Verifying Kubernetes components... + ▪ Using image docker.io/jettech/kube-webhook-certgen:v1.5.1 + ▪ Using image k8s.gcr.io/ingress-nginx/controller:v0.44.0 + ▪ Using image gcr.io/k8s-minikube/storage-provisioner:v5 + ▪ Using image docker.io/jettech/kube-webhook-certgen:v1.5.1 +🔎 Verifying ingress addon... +🌟 Enabled addons: storage-provisioner, default-storageclass, ingress +🏄 Done! kubectl is now configured to use "minikube" cluster and "default" namespace by default ``` -Then create a file named `my-awx.yml` with the following contents: +Once Minikube is deployed, check if the node(s) and `kube-apiserver` communication is working as expected. + +```bash +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +minikube Ready control-plane,master 6m28s v1.20.2 + +$ kubectl get pods -A +NAMESPACE NAME READY STATUS RESTARTS AGE +ingress-nginx ingress-nginx-admission-create-tjk94 0/1 Completed 0 6m4s +ingress-nginx ingress-nginx-admission-patch-r4pl6 0/1 Completed 0 6m4s +ingress-nginx ingress-nginx-controller-5d88495688-sbtp9 1/1 Running 0 6m4s +kube-system coredns-74ff55c5b-2wz6n 1/1 Running 0 6m4s +kube-system etcd-minikube 1/1 Running 0 6m13s +kube-system kube-apiserver-minikube 1/1 Running 0 6m13s +kube-system kube-controller-manager-minikube 1/1 Running 0 6m13s +kube-system kube-flannel-ds-amd64-lw7lv 1/1 Running 0 6m3s +kube-system kube-proxy-lcxx7 1/1 Running 0 6m3s +kube-system kube-scheduler-minikube 1/1 Running 0 6m13s +kube-system storage-provisioner 1/1 Running 1 6m17s +``` + +Now you need to deploy AWX Operator into your cluster. Start by going to https://github.com/ansible/awx-operator/releases and making note of the latest release. Replace `` in the URL `https://raw.githubusercontent.com/ansible/awx-operator//deploy/awx-operator.yaml` with the version you are deploying. + +> For this example, we are using the `devel` tag which points to the latest nightly development version. + +```bash +$ kubectl apply -f https://raw.githubusercontent.com/ansible/awx-operator/devel/deploy/awx-operator.yaml +customresourcedefinition.apiextensions.k8s.io/awxs.awx.ansible.com created +customresourcedefinition.apiextensions.k8s.io/awxbackups.awx.ansible.com created +customresourcedefinition.apiextensions.k8s.io/awxrestores.awx.ansible.com created +clusterrole.rbac.authorization.k8s.io/awx-operator created +clusterrolebinding.rbac.authorization.k8s.io/awx-operator created +serviceaccount/awx-operator created +deployment.apps/awx-operator created +``` + +Wait a few minutes and you should have the `awx-operator` running. + +```bash +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +awx-operator-7dbf9db9d7-z9hqx 1/1 Running 0 50s +``` + +Then create a file named `awx-demo.yml` with the suggested content. The `metadata.name` you provide, will be the name of the resulting AWX deployment. If you deploy more than one AWX instance to the same namespace, be sure to use unique names. ```yaml --- apiVersion: awx.ansible.com/v1beta1 kind: AWX metadata: - name: awx + name: awx-demo +spec: + service_type: nodeport + ingress_type: none + hostname: awx-demo.example.com ``` -> The metadata.name you provide, will be the name of the resulting AWX deployment. If you deploy more than one to the same namespace, be sure to use unique names. - Finally, use `kubectl` to create the awx instance in your cluster: ```bash -#> kubectl apply -f my-awx.yml +$ kubectl apply -f awx-demo.yml +awx.awx.ansible.com/awx-demo created ``` After a few minutes, the new AWX instance will be deployed. One can look at the operator pod logs in order to know where the installation process is at. This can be done by running the following command: `kubectl logs -f deployments/awx-operator`. -Once deployed, the AWX instance will be accessible at `http://awx.mycompany.com/` (assuming your cluster has an Ingress controller configured). +```bash +$ kubectl get pods -l "app.kubernetes.io/managed-by=awx-operator" +NAME READY STATUS RESTARTS AGE +awx-demo-77d96f88d5-pnhr8 4/4 Running 0 3m24s +awx-demo-postgres-0 1/1 Running 0 3m34s + +$ kubectl get svc -l "app.kubernetes.io/managed-by=awx-operator" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +awx-demo-postgres ClusterIP None 5432/TCP 4m4s +awx-demo-service NodePort 10.109.40.38 80:31006/TCP 3m56s +``` + +Once deployed, the AWX instance will be accessible by the command `minikube service awx-demo-service --url`. By default, the admin user is `admin` and the password is available in the `-admin-password` secret. To retrieve the admin password, run `kubectl get secret -admin-password -o jsonpath="{.data.password}" | base64 --decode` +You just completed the most basic install of an AWX instance via this operator. Congratulations!!!! -You just completed the most basic install of an AWX instance via this operator. Congratulations ! +For an example using the Nginx Controller in Minukube, don't miss our [demo video](https://asciinema.org/a/416946). + +[![asciicast](https://raw.githubusercontent.com/ansible/awx-operator/devel/docs/awx-demo.svg)](https://asciinema.org/a/416946) ### Admin user account configuration @@ -117,93 +197,109 @@ stringData: ### Network and TLS Configuration -#### Ingress Type +#### Service Type -By default, the AWX operator is not opinionated and won't force a specific ingress type on you. So, if `ingress_type` is not specified as part of the Custom Resource specification, it will default to `none` and nothing ingress-wise will be created. +If the `service_type` is not specified, the `ClusterIP` service will be used for your AWX Tower service. -The AWX operator provides support for four kinds of `Ingress` to access AWX: `Ingress`, `Route`, `LoadBalancer` and `NodePort`, To toggle between these options, you can add the following to your AWX CR: +The `service_type` supported options are: `ClusterIP`, `LoadBalancer` and `NodePort`. - * Route - -```yaml ---- -spec: - ... - ingress_type: Route -``` - - * Ingress - -```yaml ---- -spec: - ... - ingress_type: Ingress - hostname: awx.mycompany.com -``` - - * LoadBalancer - -```yaml ---- -spec: - ... - ingress_type: LoadBalancer - loadbalancer_protocol: http -``` - - * NodePort - -```yaml ---- -spec: - ... - ingress_type: NodePort -``` - -The AWX `Service` that gets created will have a `type` set based on the `ingress_type` being used: - -| Ingress Type `ingress_type` | Service Type | -| ------------------------------------- | -------------- | -| `LoadBalancer` | `LoadBalancer` | -| `NodePort` | `NodePort` | -| `Ingress` or `Route` or not specified | `ClusterIP` | - -#### TLS Termination - - * Route - -The following variables are customizable to specify the TLS termination procedure when `Route` is picked as an Ingress +The following variables are customizable for any `service_type` | Name | Description | Default | | ------------------------------------- | --------------------------------------------- | --------------------------------- | -| route_host | Common name the route answers for | Empty string | -| route_tls_termination_mechanism | TLS Termination mechanism (Edge, Passthrough) | Edge | -| route_tls_secret | Secret that contains the TLS information | Empty string | +| service_labels | Add custom labels | Empty string | - * Ingress - -The following variables are customizable to specify the TLS termination procedure when `Ingress` is picked as an Ingress - -| Name | Description | Default | -| -------------------------- | ---------------------------------------- | ------------- | -| ingress_annotations | Ingress annotations | Empty string | -| ingress_tls_secret | Secret that contains the TLS information | Empty string | +```yaml +--- +spec: + ... + service_type: ClusterIP + service_labels: | + environment: testing +``` * LoadBalancer -The following variables are customizable to specify the TLS termination procedure when `LoadBalancer` is picked as an Ingress +The following variables are customizable only when `service_type=LoadBalancer` | Name | Description | Default | | ------------------------------ | ---------------------------------------- | ------------- | -| loadbalancer_annotations | LoadBalancer annotations | Empty string | -| loadbalancer_protocol | Protocol to use for Loadbalancer ingress | http | -| loadbalancer_port | Port used for Loadbalancer ingress | 80 | +| loadbalancer_annotations | LoadBalancer annotations | Empty string | +| loadbalancer_protocol | Protocol to use for Loadbalancer ingress | http | +| loadbalancer_port | Port used for Loadbalancer ingress | 80 | + +```yaml +--- +spec: + ... + service_type: LoadBalancer + loadbalancer_protocol: https + loadbalancer_port: 443 + loadbalancer_annotations: | + environment: testing + service_labels: | + environment: testing +``` When setting up a Load Balancer for HTTPS you will be required to set the `loadbalancer_port` to move the port away from `80`. The HTTPS Load Balancer also uses SSL termination at the Load Balancer level and will offload traffic to AWX over HTTP. +#### Ingress Type + +By default, the AWX operator is not opinionated and won't force a specific ingress type on you. So, when the `ingress_type` is not specified, it will default to `none` and nothing ingress-wise will be created. + +The `ingress_type` supported options are: `none`, `ingress` and `route`. To toggle between these options, you can add the following to your AWX CRD: + + * None + +```yaml +--- +spec: + ... + ingress_type: none +``` + + * Generic Ingress Controller + +The following variables are customizable when `ingress_type=ingress`. The `ingress` type creates an Ingress resource as [documented](https://kubernetes.io/docs/concepts/services-networking/ingress/) which can be shared with many other Ingress Controllers as [listed](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/). + +| Name | Description | Default | +| -------------------------- | ---------------------------------------- | ---------------------------- | +| ingress_annotations | Ingress annotations | Empty string | +| ingress_tls_secret | Secret that contains the TLS information | Empty string | +| hostname | Define the FQDN | {{ meta.name }}.example.com | + +```yaml +--- +spec: + ... + ingress_type: ingress + hostname: awx-demo.example.com + ingress_annotations: | + environment: testing +``` + + * Route + +The following variables are customizable when `ingress_type=route` + +| Name | Description | Default | +| ------------------------------------- | --------------------------------------------- | --------------------------------------------------------| +| route_host | Common name the route answers for | `--` | +| route_tls_termination_mechanism | TLS Termination mechanism (Edge, Passthrough) | Edge | +| route_tls_secret | Secret that contains the TLS information | Empty string | + +```yaml +--- +spec: + ... + ingress_type: route + route_host: awx-demo.example.com + route_tls_termination_mechanism: Passthrough + route_tls_secret: custom-route-tls-secret-name +``` + ### Database Configuration #### External PostgreSQL Service @@ -572,7 +668,7 @@ There are a few moving parts to this project: 2. The `awx-operator.yaml` Kubernetes manifest file which initially deploys the Operator into a cluster. 3. Then use the command below to generate a list of commits between the versions. ```sh - #> git log --pretty="- %s (%an) - %h " .. | grep -v Merge + #> git log --no-merges --pretty="- %s (%an) - %h " .. ``` Each of these must be appropriately built in preparation for a new tag: diff --git a/ansible/group_vars/all b/ansible/group_vars/all index fb44aacd..3c3e7086 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -1,3 +1,4 @@ operator_image: quay.io/ansible/awx-operator operator_version: 0.9.0 pull_policy: Always +ansible_debug_logs: "false" diff --git a/ansible/instantiate-awx-deployment.yml b/ansible/instantiate-awx-deployment.yml index 036ae89c..83a2ed5c 100644 --- a/ansible/instantiate-awx-deployment.yml +++ b/ansible/instantiate-awx-deployment.yml @@ -20,7 +20,8 @@ spec: admin_user: admin admin_email: admin@localhost - ingress_type: "{{ ingress_type | default(omit) }}" # Either Route, Ingress or LoadBalancer + service_type: "{{ service_type | default(omit) }}" # Either clusterIP, Loadbalancer or NodePort + ingress_type: "{{ ingress_type | default(omit) }}" # Either none, Ingress, Route image: "{{ image | default(omit) }}" image_version: "{{ image_version | default(omit) }}" development_mode: "{{ development_mode | default(omit) | bool }}" diff --git a/ansible/templates/crd.yml.j2 b/ansible/templates/crd.yml.j2 index d61729d0..c5959286 100644 --- a/ansible/templates/crd.yml.j2 +++ b/ansible/templates/crd.yml.j2 @@ -70,6 +70,16 @@ spec: extra_volumes: description: Specify extra volumes to add to the application pod type: string + service_type: + description: The service type to be used on the deployed instance + type: string + enum: + - LoadBalancer + - loadbalancer + - ClusterIP + - clusterip + - NodePort + - nodeport ingress_type: description: The ingress type to use to reach the deployed instance type: string @@ -79,15 +89,11 @@ spec: - ingress - Route - route - - LoadBalancer - - loadbalancer - - NodePort - - nodeport ingress_annotations: - description: Annotations to add to the ingress + description: Annotations to add to the Ingress Controller type: string ingress_tls_secret: - description: Secret where the ingress TLS secret can be found + description: Secret where the Ingress TLS secret can be found type: string loadbalancer_annotations: description: Annotations to add to the loadbalancer diff --git a/ansible/templates/operator.yml.j2 b/ansible/templates/operator.yml.j2 index 078f75f8..786478d0 100644 --- a/ansible/templates/operator.yml.j2 +++ b/ansible/templates/operator.yml.j2 @@ -35,6 +35,8 @@ spec: value: explicit - name: OPERATOR_VERSION value: "{{ operator_version }}" + - name: ANSIBLE_DEBUG_LOGS + value: "{{ ansible_debug_logs|lower | default('false'|lower) }}" livenessProbe: httpGet: path: /healthz diff --git a/deploy/awx-operator.yaml b/deploy/awx-operator.yaml index cebbd4c7..1326035e 100644 --- a/deploy/awx-operator.yaml +++ b/deploy/awx-operator.yaml @@ -72,6 +72,16 @@ spec: extra_volumes: description: Specify extra volumes to add to the application pod type: string + service_type: + description: The service type to be used on the deployed instance + type: string + enum: + - LoadBalancer + - loadbalancer + - ClusterIP + - clusterip + - NodePort + - nodeport ingress_type: description: The ingress type to use to reach the deployed instance type: string @@ -81,15 +91,11 @@ spec: - ingress - Route - route - - LoadBalancer - - loadbalancer - - NodePort - - nodeport ingress_annotations: - description: Annotations to add to the ingress + description: Annotations to add to the Ingress Controller type: string ingress_tls_secret: - description: Secret where the ingress TLS secret can be found + description: Secret where the Ingress TLS secret can be found type: string loadbalancer_annotations: description: Annotations to add to the loadbalancer @@ -621,7 +627,7 @@ spec: serviceAccountName: awx-operator containers: - name: awx-operator - image: "quay.io/ansible/awx-operator:0.9.0" + image: "quay.io/ansible/awx-operator:devel" imagePullPolicy: "Always" volumeMounts: - mountPath: /tmp/ansible-operator/runner @@ -639,7 +645,9 @@ spec: - name: ANSIBLE_GATHERING value: explicit - name: OPERATOR_VERSION - value: "0.9.0" + value: "devel" + - name: ANSIBLE_DEBUG_LOGS + value: "false" livenessProbe: httpGet: path: /healthz diff --git a/deploy/crds/awx_v1beta1_crd.yaml b/deploy/crds/awx_v1beta1_crd.yaml index d61729d0..c5959286 100644 --- a/deploy/crds/awx_v1beta1_crd.yaml +++ b/deploy/crds/awx_v1beta1_crd.yaml @@ -70,6 +70,16 @@ spec: extra_volumes: description: Specify extra volumes to add to the application pod type: string + service_type: + description: The service type to be used on the deployed instance + type: string + enum: + - LoadBalancer + - loadbalancer + - ClusterIP + - clusterip + - NodePort + - nodeport ingress_type: description: The ingress type to use to reach the deployed instance type: string @@ -79,15 +89,11 @@ spec: - ingress - Route - route - - LoadBalancer - - loadbalancer - - NodePort - - nodeport ingress_annotations: - description: Annotations to add to the ingress + description: Annotations to add to the Ingress Controller type: string ingress_tls_secret: - description: Secret where the ingress TLS secret can be found + description: Secret where the Ingress TLS secret can be found type: string loadbalancer_annotations: description: Annotations to add to the loadbalancer diff --git a/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml b/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml index 953c30db..721b5653 100644 --- a/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/awx-operator/manifests/awx-operator.clusterserviceversion.yaml @@ -178,46 +178,51 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - - displayName: Ingress Type + - displayName: Tower Service Type + path: service_type + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:select:ClusterIP + - urn:alm:descriptor:com.tectonic.ui:select:LoadBalancer + - urn:alm:descriptor:com.tectonic.ui:select:NodePort + - displayName: Tower Ingress Type path: ingress_type x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:select:none - urn:alm:descriptor:com.tectonic.ui:select:Ingress - urn:alm:descriptor:com.tectonic.ui:select:Route - - urn:alm:descriptor:com.tectonic.ui:select:LoadBalancer - - urn:alm:descriptor:com.tectonic.ui:select:NodePort - - displayName: Ingress Annotations + - displayName: Tower Ingress Annotations path: ingress_annotations x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:Ingress - - displayName: Ingress TLS Secret + - displayName: Tower Ingress TLS Secret path: ingress_tls_secret x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:io.kubernetes:Secret - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:Ingress - - displayName: LoadBalancer Annotations + - displayName: Tower LoadBalancer Annotations path: loadbalancer_annotations x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:LoadBalancer - - displayName: LoadBalancer Protocol + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer + - displayName: Tower LoadBalancer Protocol path: loadbalancer_protocol x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:select:http - urn:alm:descriptor:com.tectonic.ui:select:https - - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:LoadBalancer - - displayName: LoadBalancer Port + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer + - displayName: Tower LoadBalancer Port path: loadbalancer_port x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:number - - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:LoadBalancer + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:service_type:LoadBalancer - displayName: Route DNS host path: route_host x-descriptors: diff --git a/docs/awx-demo.svg b/docs/awx-demo.svg new file mode 100644 index 00000000..8414086e --- /dev/null +++ b/docs/awx-demo.svg @@ -0,0 +1 @@ +$$#Forthisdemo,I'mrunningitonFedora34withkvm2$#Pleasenotetheoptionspassedtotheminikubestartcommand$#shallbedifferentforeachenvironment.$#RefertotheMinikubedocumentationforfurtherassistance$#Tospeedthingsuponthisvideo,hereisthecommandIusedtostartmyminikubeversion$#minikubestart--driver=kvm2--addons=ingress--cpus=4--cni=flannel\$#--install-addons=true--kubernetes-version=stable--memory=12g$minikubestart😄minikubev1.20.0onFedora34✨Usingthekvm2driverbasedonexistingprofile👍Startingcontrolplanenodeminikubeinclusterminikube🏃Updatingtherunningkvm2"minikube"VM...🐳PreparingKubernetesv1.20.2onDocker20.10.6...🔎VerifyingKubernetescomponents...▪Usingimagek8s.gcr.io/ingress-nginx/controller:v0.44.0▪Usingimagedocker.io/jettech/kube-webhook-certgen:v1.5.1▪Usingimagegcr.io/k8s-minikube/storage-provisioner:v5🔎Verifyingingressaddon...🌟Enabledaddons:storage-provisioner,default-storageclass,ingress🏄Done!kubectlisnowconfiguredtouse"minikube"clusterand"default"namespacebydefault$#Nowlet'scheckifthepodsareontherunningstate.$#Itmighttakeafewsecondstogetallofthemupandrunningonyourend$kubectlgetnodesNAMESTATUSROLESAGEVERSIONminikubeReadycontrol-plane,master45mv1.20.2$kubectlgetpods-ANAMESPACENAMEREADYSTATUSRESTARTSAGEingress-nginxingress-nginx-admission-create-jws4j0/1Completed044mingress-nginxingress-nginx-admission-patch-sjk7b0/1Completed044mingress-nginxingress-nginx-controller-5d88495688-fpfnk1/1Running044mkube-systemcoredns-74ff55c5b-xcpzh1/1Running044mkube-systemetcd-minikube1/1Running045mkube-systemkube-apiserver-minikube1/1Running045mkube-systemkube-controller-manager-minikube1/1Running045mkube-systemkube-flannel-ds-amd64-v7c7g1/1Running044mkube-systemkube-proxy-8mpm91/1Running044mkube-systemkube-scheduler-minikube1/1Running045mkube-systemstorage-provisioner1/1Running145m$#Forthisdemo,I'mdeployingtheawx-operatorfromthedevelbranch$#butpleaserefertothedocumentationonhowdeployastablerelease$kubectlapply-fawx-operator.yamlcustomresourcedefinition.apiextensions.k8s.io/awxs.awx.ansible.comcreatedcustomresourcedefinition.apiextensions.k8s.io/awxbackups.awx.ansible.comcreatedcustomresourcedefinition.apiextensions.k8s.io/awxrestores.awx.ansible.comcreatedclusterrole.rbac.authorization.k8s.io/awx-operatorcreatedclusterrolebinding.rbac.authorization.k8s.io/awx-operatorcreatedserviceaccount/awx-operatorcreateddeployment.apps/awx-operatorcreated$#Let'swaitthecontainerimagetobedownloadandtheawx-operator$#shouldbeinarunningstate$kubectl$kubectlget$kubectlgetpods$kubectlgetpods-wNAMEREADYSTATUSRESTARTSAGEawx-operator-7dbf9db9d7-z62pm1/1Running012s^C^C$$#Nowfirst,let'screateaAWXinstancewithusingaNodePortservice$#Thisprocesswilltakeafewminutestofinish,butweshouldhavea$#PostgreSQLstatefulsetrunningandaAWXpodwith4containersinRunningstate$catawx-demo.yml---apiVersion:awx.ansible.com/v1beta1kind:AWXmetadata:name:awx-demospec:service_type:nodeporthostname:awx-demo.example.com$kubectlapply-fawx-demo.ymlawx.awx.ansible.com/awx-democreated$#Inthemeantime,youcaninspecttheawx-operatorlogsaswell$#kubectllogs-fdeploy/awx-operator$#Allowsometimetogettheimagespulledandforthedatabasemigration$#Gogetacoffee:)$kubectlgetpods-l"app.kubernetes.io/managed-by=awx-operator"-wNAMEREADYSTATUSRESTARTSAGEawx-demo-postgres-00/1Pending00sawx-demo-postgres-00/1ContainerCreating00sawx-demo-postgres-01/1Running02sawx-demo-77d96f88d5-rbqz90/4Pending00sawx-demo-77d96f88d5-rbqz90/4ContainerCreating00sawx-demo-77d96f88d5-rbqz94/4Running05s$#Nowthateverythingisintherunningstate,weshouldbegood:)$kubectl get pods -l "app.kubernetes.io/managed-by=awx-operator"$kubectlgetpods-l"app.kubernetes.io/managed-by=awx-operator"NAMEREADYSTATUSRESTARTSAGEawx-demo-77d96f88d5-rbqz94/4Running018sawx-demo-postgres-01/1Running028s$kubectl get svc -l "app.kubernetes.io/managed-by=awx-operator"$kubectlgetsvc-l"app.kubernetes.io/managed-by=awx-operator"NAMETYPECLUSTER-IPEXTERNAL-IPPORT(S)AGEawx-demo-postgresClusterIPNone<none>5432/TCP35sawx-demo-serviceNodePort10.98.233.228<none>80:32747/TCP27s$#Let'sdoaquicktesttocheckifthingsarelookinggood$kubectliexecawx-demo/bin/bashUsethearrowkeystonavigate:↓↑→←?SelectPod:Namespace:default|Pod:▸awx-demo-77d96f88d5-rbqz9Namespace:default|Pod:awx-demo-postgres-0Namespace:default|Pod:✔awx-demo-77d96f88d5-rbqz9?SelectContainer:Container:▸redisContainer:awx-demo-webContainer:awx-demo-taskContainer:awx-demo-eeContainer:redisContainer:▸awx-demo-webContainer:▸awx-demo-taskContainer:✔awx-demo-taskbash-4.4$bash-4.4$awx-manageversion19.1.0bash-4.4$awx-managecheck_dbDatabaseVersion:PostgreSQL12.7(Debian12.7-1.pgdg100+1)onx86_64-pc-linux-gnu,compiledbygcc(Debian8.3.0-6)8.3.0,64-bitbash-4.4$exitexit$#Let'stovisittheNodePortURLusingtheminikubeservicecommand$minikube service list$minikubeservicelist|---------------|------------------------------------|--------------|----------------------------||NAMESPACE|NAME|TARGETPORT|URL||default|awx-demo-postgres|Nonodeport||default|awx-demo-service|http/80|http://192.168.39.79:32747||default|default|awx-operator-metrics|Nonodeport|||default|kubernetes|Nonodeport||ingress-nginx|ingress-nginx-controller|http/80|http://192.168.39.79:32732||||https/443|http://192.168.39.79:30947||ingress-nginx|ingress-nginx-controller-admission|Nonodeport||kube-system|kube-dns|Nonodeport|$minikubeserviceawx-demo-service--urlhttp://192.168.39.79:32747$#$#L$#Le$#Let$#Let'$#Let's$#Let'scheck$#Let'scheckthe$#Let'schecktheAPI$#Let'schecktheAPInow$curl-L-q$(minikubeserviceawx-demo-service--url)/api/v2/ping2>/dev/null|python-mjson.tool{"ha":false,"version":"19.1.0","active_node":"awx-demo-77d96f88d5-rbqz9","install_uuid":"352415ad-564e-41cc-bbc2-85bd9169342e","instances":[{"node":"awx-demo-77d96f88d5-rbqz9","uuid":"00000000-0000-0000-0000-000000000000","heartbeat":"2021-05-25T05:00:46.328860Z","capacity":0,"version":"19.1.0"}],"instance_groups":["name":"tower","instances":[]]}$#S$#Sweet!!$#Sweet!!You$#Sweet!!Yougot$#Sweet!!Yougotit!!$#Sweet!!Yougotit!!:)$#Nowlet'sdeployanotherAWXinstanceusingaNGINXIngressController$#OncewehaveitinaRunningstate,wenowhaveaningressresourcecreated$catawx-nginx-ingress.ymlname:awx-nginxservice_type:clusteripingress_type:ingresshostname:my-awx.example.com$kubectlapply-fawx-nginx-ingress.ymlawx.awx.ansible.com/awx-nginxcreated$kubectlgetawxNAMEAGEawx-demo2m22sawx-nginx4sawx-demo-77d96f88d5-rbqz94/4Running02mawx-demo-postgres-01/1Running02m10sawx-nginx-postgres-00/1Pending00sawx-nginx-postgres-00/1ContainerCreating00sawx-nginx-postgres-01/1Running03sawx-nginx-cb66586f-m6qrj0/4Pending00sawx-nginx-cb66586f-m6qrj0/4ContainerCreating00sawx-nginx-cb66586f-m6qrj4/4Running05s$#Now$#Nowwe$#Nowwehave$#Nowwehavethe$#Nowwehavetheservices$#Nowwehavetheservicescreated$#Nowwehavetheservicescreatedand$#Nowwehavetheservicescreatedandthe$#Nowwehavetheservicescreatedandthepods$#Nowwehavetheservicescreatedandthepodsrunningawx-demo-77d96f88d5-rbqz94/4Running02m43sawx-demo-postgres-01/1Running02m53sawx-nginx-cb66586f-m6qrj4/4Running027sawx-nginx-postgres-01/1Running037sNAMETYPECLUSTER-IPEXTERNAL-IPPORT(S)AGEawx-demo-postgresClusterIPNone<none>5432/TCP2m55sawx-demo-serviceNodePort10.98.233.228<none>80:32747/TCP2m47sawx-nginx-postgresClusterIPNone<none>5432/TCP39sawx-nginx-serviceClusterIP10.100.157.46<none>80/TCP31s|default|awx-nginx-postgres|Nonodeport||default|awx-nginx-service|Nonodeport|$#Let'snow$#Let'snowinspect$#Let'snowinspectthe$#Let'snowinspecttheingress$kubectlgetingressesNAMECLASSHOSTSADDRESSPORTSAGEawx-nginx-ingress<none>my-awx.example.com192.168.39.798054s$kubectldescribeingressawx-nginx-ingressName:awx-nginx-ingressNamespace:defaultAddress:192.168.39.79Defaultbackend:default-http-backend:80(<error:endpoints"default-http-backend"notfound>)Rules:HostPathBackends----------------my-awx.example.com/awx-nginx-service:80(10.244.0.19:8052)Annotations:<none>Events:TypeReasonAgeFromMessage-------------------------NormalSync56s(x2over58s)nginx-ingress-controllerScheduledforsync$#Withtheingressruleinplace,weshouldbeabletoaccessit$#Forthat,let'saddtheentryonthe/etc/hosts$kubectlgetingressesawx-nginx-ingress|grep-vNAME|awk-F'''{print$4""$3}'192.168.39.79my-awx.example.com$kubectlgetingressesawx-nginx-ingress|grep-vNAME|awk-F'''{print$4""$3}'|sudotee-a/etc/hosts$curl-L-qhttp://my-awx.example.com/api/v2/ping2>/dev/null|python-mjson.tool"active_node":"awx-nginx-cb66586f-m6qrj","install_uuid":"3f21fe54-5f12-4546-8200-b8976cca45f9","node":"awx-nginx-cb66586f-m6qrj","heartbeat":"2021-05-25T05:03:02.224788Z",$#ThanksforwatchingandhappyAnsibleautomation!$# For this demo, I'm running it on Fedora 34 with kvm2$# Please note the options passed to the minikube start command$# shall be different for each environment.$# Refer to the Minikube documentation for further assistance$# To speed things up on this video, here is the command I used to start my minikube version$# minikube start --driver=kvm2 --addons=ingress --cpus=4 --cni=flannel \$# --install-addons=true --kubernetes-version=stable --memory=12g$minikube start🐳PreparingKubernetesv1.20.2onDocker20.10.6...|🐳PreparingKubernetesv1.20.2onDocker20.10.6.../🐳PreparingKubernetesv1.20.2onDocker20.10.6...-🐳PreparingKubernetesv1.20.2onDocker20.10.6...\$# Now let's check if the pods are on the running state.$# It might take a few seconds to get all of them up and running on your end$kubectl get nodes$kubectl get pods -Aingress-nginxkube-systemkube-systemstorage-provisioner1/1$# For this demo, I'm deploying the awx-operator from the devel branch$# but please refer to the documentation on how deploy a stable release$kubectl apply -f awx-operator.yaml$# Let's wait the container image to be download and the awx-operator$# should be in a running state$k$ku$kub$kube$kubec$kubect$kubectlg$kubectlge$kubectlgetp$kubectlgetpo$kubectlgetpod$kubectlgetpods-$# Now first, let's create a AWX instance with using a NodePort service$# This process will take a few minutes to finish, but we should have a$# PostgreSQL statefulset running and a AWX pod with 4 containers in Running state$cat awx-demo.yml$kubectl apply -f awx-demo.yml$# In the meantime, you can inspect the awx-operator logs as well$# kubectl logs -f deploy/awx-operator$# Allow some time to get the images pulled and for the database migration$# Go get a coffee :)$kubectl get pods -l "app.kubernetes.io/managed-by=awx-operator" -w$# Now that everything is in the running state, we should be good :)$# Let's do a quick test to check if things are looking good$kubectl iexec awx-demo /bin/bashbash-4.4$ebash-4.4$exbash-4.4$exi$# Let's to visit the NodePort URL using the minikube service command|NAMESPACE|NAME||default|awx-demo-service|http/80|||https/443||kube-system$minikube service awx-demo-service --url$#Let'sc$#Let'sch$#Let'sche$#Let'schec$#Let'scheckt$#Let'scheckth$#Let'schecktheA$#Let'schecktheAP$#Let'schecktheAPIn$#Let'schecktheAPIno$curl -L -q $(minikube service awx-demo-service --url)/api/v2/ping 2>/dev/null | python -m json.tool$#Se$#Sw$#Swe$#Swee$#Sweet$#Sweet!$#Sweet!!Y$#Sweet!!Yo$#Sweet!!Youg$#Sweet!!Yougo$#Sweet!!Yougoti$#Sweet!!Yougotit$#Sweet!!Yougotit!$#Sweet!!Yougotit!!:$# Now let's deploy another AWX instance using a NGINX Ingress Controller$# Once we have it in a Running state, we now have an ingress resource created$cat awx-nginx-ingress.yml$kubectl apply -f awx-nginx-ingress.yml$kubectl get awx$kubectl get pods -l "app.kubernetes.io/managed-by=awx-operator" -w$#N$#No$#Noww$#Nowweh$#Nowweha$#Nowwehav$#Nowwehavet$#Nowwehaveth$#Nowwehavethes$#Nowwehavethese$#Nowwehavetheser$#Nowwehavetheserv$#Nowwehavetheservi$#Nowwehavetheservic$#Nowwehavetheservice$#Nowwehavetheservicesc$#Nowwehavetheservicescr$#Nowwehavetheservicescre$#Nowwehavetheservicescrea$#Nowwehavetheservicescreat$#Nowwehavetheservicescreate$#Nowwehavetheservicescreateda$#Nowwehavetheservicescreatedan$#Nowwehavetheservicescreatedandt$#Nowwehavetheservicescreatedandth$#Nowwehavetheservicescreatedandthep$#Nowwehavetheservicescreatedandthepo$#Nowwehavetheservicescreatedandthepod$#Nowwehavetheservicescreatedandthepodsr$#Nowwehavetheservicescreatedandthepodsru$#Nowwehavetheservicescreatedandthepodsrun$#Nowwehavetheservicescreatedandthepodsrunn$#Nowwehavetheservicescreatedandthepodsrunni$#Nowwehavetheservicescreatedandthepodsrunninawx-demo-serviceNodePort|default|awx-demo-postgres||default|awx-nginx-service||default|awx-operator-metrics||default|kubernetes||ingress-nginx|ingress-nginx-controller|http/80||ingress-nginx|ingress-nginx-controller-admission|Nonodeport|---------------|------------------------------------|$#Let'sn$#Let'sno$#Let'snowi$#Let'snowin$#Let'snowins$#Let'snowinsp$#Let'snowinspe$#Let'snowinspec$#Let'snowinspectt$#Let'snowinspectth$#Let'snowinspectthei$#Let'snowinspectthein$#Let'snowinspecttheing$#Let'snowinspecttheingr$#Let'snowinspecttheingre$#Let'snowinspecttheingres$kubectl get ingresses$kubectl describe ingress awx-nginx-ingress$# With the ingress rule in place, we should be able to access it$# For that, let's add the entry on the$#Forthat,let'saddtheentryonthe$#Forthat,let'saddtheentryonthe/$#Forthat,let'saddtheentryonthe/e$#Forthat,let'saddtheentryonthe/et$#Forthat,let'saddtheentryonthe/etc$#Forthat,let'saddtheentryonthe/etc/$#Forthat,let'saddtheentryonthe/etc/h$#Forthat,let'saddtheentryonthe/etc/ho$#Forthat,let'saddtheentryonthe/etc/hos$#Forthat,let'saddtheentryonthe/etc/host$kubectl get ingresses awx-nginx-ingress | grep -v NAME | awk -F' ' '{print $4 " " $3 }'$kubectl get ingresses awx-nginx-ingress | grep -v NAME | awk -F' ' '{print $4 " " $3 }' | sudo tee -a /etc/hosts$curl -L -q http://my-awx.example.com/api/v2/ping 2>/dev/null | python -m json.tool$# Thanks for watching and happy Ansible automation! \ No newline at end of file diff --git a/molecule/test-local/converge.yml b/molecule/test-local/converge.yml index ee5f1a35..9ad1d895 100644 --- a/molecule/test-local/converge.yml +++ b/molecule/test-local/converge.yml @@ -28,6 +28,7 @@ pull_policy: Never operator_image: awx.ansible.com/awx-operator operator_version: testing + ansible_debug_logs: "true" custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_molecule.yaml'])) | from_yaml }}" tasks: diff --git a/molecule/test-minikube/converge.yml b/molecule/test-minikube/converge.yml index 2f9ec440..a1adbeb3 100644 --- a/molecule/test-minikube/converge.yml +++ b/molecule/test-minikube/converge.yml @@ -36,6 +36,7 @@ pull_policy: Never operator_image: awx.ansible.com/awx-operator operator_version: testing + ansible_debug_logs: "true" # Change this to _awx to test AWX, _tower to test Tower. custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/awx_v1beta1_molecule.yaml'])) | from_yaml }}" diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index da585001..0d2334ad 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -7,6 +7,7 @@ database_name: "{{ deployment_type }}" database_username: "{{ deployment_type }}" task_privileged: false +service_type: ClusterIP ingress_type: none # Add annotations to the service account. Specify as literal block. E.g.: @@ -51,7 +52,7 @@ route_tls_secret: '' # route_host: '' -hostname: '{{ deployment_type }}.example.com' +hostname: '{{ meta.name }}.example.com' # Add a nodeSelector for the AWX pods. It must match a node's labels for the pod # to be scheduled on that node. Specify as literal block. E.g.: diff --git a/roles/installer/templates/ingress.yaml.j2 b/roles/installer/templates/ingress.yaml.j2 index c8df3135..6bd586c4 100644 --- a/roles/installer/templates/ingress.yaml.j2 +++ b/roles/installer/templates/ingress.yaml.j2 @@ -1,4 +1,4 @@ -{% if 'ingress' == ingress_type|lower %} +{% if ingress_type|lower == "ingress" %} --- apiVersion: extensions/v1beta1 kind: Ingress @@ -32,7 +32,7 @@ spec: {% endif %} {% endif %} -{% if 'route' == ingress_type|lower %} +{% if ingress_type|lower == "route" %} --- apiVersion: route.openshift.io/v1 kind: Route diff --git a/roles/installer/templates/service.yaml.j2 b/roles/installer/templates/service.yaml.j2 index 70a62aa5..522be843 100644 --- a/roles/installer/templates/service.yaml.j2 +++ b/roles/installer/templates/service.yaml.j2 @@ -17,7 +17,7 @@ metadata: {% endif %} spec: ports: -{% if ingress_type | lower != 'loadbalancer' and loadbalancer_protocol | lower != 'https' %} +{% if service_type | lower != 'loadbalancer' and loadbalancer_protocol | lower != 'https' %} - port: 80 protocol: TCP targetPort: 8052 @@ -29,12 +29,12 @@ spec: targetPort: 8053 name: https {% endif %} -{% if ingress_type | lower == 'loadbalancer' and loadbalancer_protocol | lower == 'https' %} +{% if service_type | lower == 'loadbalancer' and loadbalancer_protocol | lower == 'https' %} - port: {{ loadbalancer_port }} protocol: TCP targetPort: 8052 name: https -{% elif ingress_type | lower == 'loadbalancer' and loadbalancer_protocol | lower != 'https' %} +{% elif service_type | lower == 'loadbalancer' and loadbalancer_protocol | lower != 'https' %} - port: {{ loadbalancer_port }} protocol: TCP targetPort: 8052 @@ -44,9 +44,9 @@ spec: app.kubernetes.io/name: '{{ meta.name }}' app.kubernetes.io/managed-by: '{{ deployment_type }}-operator' app.kubernetes.io/component: '{{ deployment_type }}' -{% if ingress_type | lower == "loadbalancer" %} +{% if service_type | lower == "loadbalancer" %} type: LoadBalancer -{% elif ingress_type != "none" %} +{% elif service_type | lower == "nodeport" %} type: NodePort {% else %} type: ClusterIP diff --git a/scripts/build.sh b/scripts/build.sh index 24e6360d..012bbf2c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -8,7 +8,7 @@ ## ## git clone https://github.com/ansible/awx-operator.git ## cd awx-operator -## REGISTRY=registry.example.com/ansible TAG=mytag scripts/build.sh +## REGISTRY=registry.example.com/ansible TAG=mytag ANSIBLE_DEBUG_LOGS=true scripts/build.sh ## ## As a result, the $REGISTRY will be populated with 2 images ## registry.example.com/ansible/awx-operator:mytag @@ -51,6 +51,7 @@ prepare_local_deploy() { echo "operator_image: $REGISTRY/$OPERATOR_IMAGE" > ansible/group_vars/all echo "operator_version: $TAG" >> ansible/group_vars/all echo "pull_policy: Always" >> ansible/group_vars/all + echo "ansible_debug_logs: ${ANSIBLE_DEBUG_LOGS:-false}" >> ansible/group_vars/all ansible-playbook ansible/chain-operator-files.yml }