k8s_drain new module (#141)

k8s_drain new module

SUMMARY

new module to drain, cordon or uncordon node from k8s cluster.
#141

ISSUE TYPE


New Module Pull Request

COMPONENT NAME

k8s_drain

Reviewed-by: Abhijeet Kasurde <None>
Reviewed-by: None <None>
Reviewed-by: Mike Graves <mgraves@redhat.com>
Reviewed-by: None <None>

molecule/default/molecule.yml

@@ -13,7 +13,7 @@ provisioner:
   log: true
   config_options:
     inventory:
-      enable_plugins: kubernetes.core.k8s
+      enable_plugins: kubernetes.core.k8s,yaml
   lint: {}
   inventory:
     hosts:

molecule/default/prepare.yml

@@ -6,45 +6,7 @@
   collections:
     - kubernetes.core
 
-  vars:
-    node_taints:
-      - "node.kubernetes.io/not-ready"
-      - "node.kubernetes.io/unreachable"
-      - "node.kubernetes.io/unschedulable"
-    kind_bin_path: "/usr/local/bin/kind"
-
   tasks:
-    # We are spawning k8s cluster using kind executable and we ensure that the cluster is up
-    # and node is ready, if this is not validated we may face issue later on when running tests
-    - name: Check if kind binary is available or not
-      stat:
-        path: "{{ kind_bin_path }}"
-      register: r
-
-    - name: Download kind if not available
-      get_url:
-        url: https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
-        dest: "{{ kind_bin_path }}"
-      when: not r.stat.exists
-
-    - name: Make kind executable
-      file:
-        path: "{{ kind_bin_path }}"
-        mode: '0755'
-
-    - name: Check if Kind cluster exists
-      command: "{{ kind_bin_path }} get clusters"
-      register: r
-      ignore_errors: true
-
-    - name: Create cluster
-      command: "{{ kind_bin_path }} create cluster"
-      when: r.stdout == ''
-
-    - name: Assert that nodes are ready
-      k8s_info:
-        kind: Node
-      retries: 10
-      delay: 30
-      register: nodes
-      until: nodes.resources | selectattr("spec.taints", "defined") |  map(attribute="spec.taints") | list | length == 0
+    - name: Include drain.yml
+      include_tasks:
+        file: tasks/drain.yml

molecule/default/tasks/drain.yml

@@ -0,0 +1,219 @@
+---
+- block:
+  - name: Set common facts
+    set_fact:
+      drain_namespace: "drain"
+      drain_daemonset_name: "promotheus-dset"
+      drain_pod_name: "pod-drain"
+
+  - name: Create {{ drain_namespace }} namespace
+    k8s:
+      kind: Namespace
+      name: '{{ drain_namespace }}'
+
+  - name: list cluster nodes
+    k8s_info:
+      kind: node
+    register: nodes
+
+  - name: Select uncordoned nodes
+    set_fact:
+      uncordoned_nodes: "{{ nodes.resources | selectattr('spec.unschedulable', 'undefined') | map(attribute='metadata.name') | list}}"
+
+  - name: Assert that at least one node is schedulable
+    assert:
+      that:
+      - uncordoned_nodes | length > 0
+
+  - name: select node to drain
+    set_fact:
+      node_to_drain: '{{ uncordoned_nodes[0] }}'
+
+  - name: Deploy daemonset on cluster
+    k8s:
+      namespace: '{{ drain_namespace }}'
+      definition:
+        apiVersion: apps/v1
+        kind: DaemonSet
+        metadata:
+          name: '{{ drain_daemonset_name }}'
+        spec:
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                - matchFields:
+                  - key: metadata.name
+                    operator: In
+                    values:
+                    - '{{ node_to_drain }}'
+          selector:
+            matchLabels:
+              name: prometheus-exporter
+          template:
+            metadata:
+              labels:
+                name: prometheus-exporter
+            spec:
+              containers:
+              - name: prometheus
+                image: prom/node-exporter
+                ports:
+                - containerPort: 80
+
+  - name: Create Pods not managed by ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet.
+    k8s:
+      namespace: '{{ drain_namespace }}'
+      wait: yes
+      definition:
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          name: '{{ drain_pod_name }}'
+        spec:
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                - matchFields:
+                  - key: metadata.name
+                    operator: In
+                    values:
+                    - '{{ node_to_drain }}'
+          containers:
+          - name: c0
+            image: busybox
+            command:
+            - /bin/sh
+            - -c
+            - while true;do date;sleep 5; done
+
+  - name: Cordon node
+    k8s_drain:
+      state: cordon
+      name: '{{ node_to_drain }}'
+    register: cordon
+
+  - name: assert that cordon is changed
+    assert:
+      that:
+      - cordon is changed
+
+  - name: Test cordon idempotency
+    k8s_drain:
+      state: cordon
+      name: '{{ node_to_drain }}'
+    register: cordon
+
+  - name: assert that cordon is not changed
+    assert:
+      that:
+      - cordon is not changed
+
+  - name: Get pods
+    k8s_info:
+      kind: Pod
+      namespace: '{{ drain_namespace }}'
+    register: Pod
+
+  - name: assert that pods are running on cordoned node
+    assert:
+      that:
+      - "{{ Pod.resources | selectattr('status.phase', 'equalto', 'Running') | selectattr('spec.nodeName', 'equalto', node_to_drain) | list | length > 0 }}"
+
+  - name: Uncordon node
+    k8s_drain:
+      state: uncordon
+      name: '{{ node_to_drain }}'
+    register: uncordon
+
+  - name: assert that uncordon is changed
+    assert:
+      that:
+      - uncordon is changed
+
+  - name: Test uncordon idempotency
+    k8s_drain:
+      state: uncordon
+      name: '{{ node_to_drain }}'
+    register: uncordon
+
+  - name: assert that uncordon is not changed
+    assert:
+      that:
+      - uncordon is not changed
+
+  - name: Drain node
+    k8s_drain:
+      state: drain
+      name: '{{ node_to_drain }}'
+    ignore_errors: true
+    register: drain_result
+
+  - name: assert that drain failed due to DaemonSet managed Pods
+    assert:
+      that:
+      - drain_result is failed
+      - '"cannot delete DaemonSet-managed Pods" in drain_result.msg'
+      - '"cannot delete Pods not managed by ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet" in drain_result.msg'
+
+  - name: Drain node using ignore_daemonsets and force options
+    k8s_drain:
+      state: drain
+      name: '{{ node_to_drain }}'
+      delete_options:
+        force: true
+        ignore_daemonsets: true
+        wait_timeout: 0
+    register: drain_result
+
+  - name: assert that node has been drained
+    assert:
+      that:
+      - drain_result is changed
+      - '"node {{ node_to_drain }} marked unschedulable." in drain_result.result'
+
+  - name: assert that unmanaged pod were deleted
+    k8s_info:
+      namespace: '{{ drain_namespace }}'
+      kind: Pod
+    register: _result
+    failed_when: _result.resources | selectattr("metadata.ownerReferences", "undefined") | length > 0
+
+  - name: Test drain idempotency
+    k8s_drain:
+      state: drain
+      name: '{{ node_to_drain }}'
+      delete_options:
+        force: true
+        ignore_daemonsets: true
+    register: drain_result
+
+  - name: Check idempotency
+    assert:
+      that:
+      - drain_result is not changed
+
+  - name: Get DaemonSet
+    k8s_info:
+      kind: DaemonSet
+      namespace: '{{ drain_namespace }}'
+      name: '{{ drain_daemonset_name }}'
+    register: dset_result
+
+  - name: assert that daemonset managed pods were not removed
+    assert:
+      that:
+      - dset_result.resources | list | length > 0
+
+  - name: Uncordon node
+    k8s_drain:
+      state: uncordon
+      name: '{{ node_to_drain }}'
+
+  always:
+  - name: delete namespace
+    k8s:
+      state: absent
+      kind: namespace
+      name: '{{ drain_namespace }}'