Add new waiter (#306)

Add new waiter

SUMMARY

This refactors the waiter logic from common.py into a separate module.

ISSUE TYPE

COMPONENT NAME

ADDITIONAL INFORMATION

Reviewed-by: None <None>
Reviewed-by: Alina Buzachis <None>
Reviewed-by: None <None>

plugins/module_utils/k8s/waiter.py

@@ -0,0 +1,204 @@
+import time
+from functools import partial
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
+
+from ansible.module_utils.parsing.convert_bool import boolean
+
+try:
+    from kubernetes.dynamic.exceptions import NotFoundError
+    from kubernetes.dynamic.resource import Resource, ResourceField, ResourceInstance
+except ImportError:
+    # These are defined only for the sake of Ansible's checked import requirement
+    Resource = Any  # type: ignore
+    ResourceInstance = Any  # type: ignore
+    pass
+
+
+def deployment_ready(deployment: ResourceInstance) -> bool:
+    # FIXME: frustratingly bool(deployment.status) is True even if status is empty
+    # Furthermore deployment.status.availableReplicas == deployment.status.replicas == None if status is empty
+    # deployment.status.replicas is None is perfectly ok if desired replicas == 0
+    # Scaling up means that we also need to check that we're not in a
+    # situation where status.replicas == status.availableReplicas
+    # but spec.replicas != status.replicas
+    return bool(
+        deployment.status
+        and deployment.spec.replicas == (deployment.status.replicas or 0)
+        and deployment.status.availableReplicas == deployment.status.replicas
+        and deployment.status.observedGeneration == deployment.metadata.generation
+        and not deployment.status.unavailableReplicas
+    )
+
+
+def pod_ready(pod: ResourceInstance) -> bool:
+    return bool(
+        pod.status
+        and pod.status.containerStatuses is not None
+        and all(container.ready for container in pod.status.containerStatuses)
+    )
+
+
+def daemonset_ready(daemonset: ResourceInstance) -> bool:
+    return bool(
+        daemonset.status
+        and daemonset.status.desiredNumberScheduled is not None
+        and daemonset.status.updatedNumberScheduled
+        == daemonset.status.desiredNumberScheduled
+        and daemonset.status.numberReady == daemonset.status.desiredNumberScheduled
+        and daemonset.status.observedGeneration == daemonset.metadata.generation
+        and not daemonset.status.unavailableReplicas
+    )
+
+
+def statefulset_ready(statefulset: ResourceInstance) -> bool:
+    return bool(
+        statefulset.status
+        and statefulset.spec.updateStrategy.type == "RollingUpdate"
+        and statefulset.status.observedGeneration
+        == (statefulset.metadata.generation or 0)
+        and statefulset.status.updateRevision == statefulset.status.currentRevision
+        and statefulset.status.updatedReplicas == statefulset.spec.replicas
+        and statefulset.status.readyReplicas == statefulset.spec.replicas
+        and statefulset.status.replicas == statefulset.spec.replicas
+    )
+
+
+def custom_condition(condition: Dict, resource: ResourceInstance) -> bool:
+    if not resource.status or not resource.status.conditions:
+        return False
+    matches = [x for x in resource.status.conditions if x.type == condition["type"]]
+    if not matches:
+        return False
+    # There should never be more than one condition of a specific type
+    match: ResourceField = matches[0]
+    if match.status == "Unknown":
+        if match.status == condition["status"]:
+            if "reason" not in condition:
+                return True
+            if condition["reason"]:
+                return match.reason == condition["reason"]
+        return False
+    status = True if match.status == "True" else False
+    if status == boolean(condition["status"], strict=False):
+        if condition.get("reason"):
+            return match.reason == condition["reason"]
+        return True
+    return False
+
+
+def resource_absent(resource: ResourceInstance) -> bool:
+    return not exists(resource)
+
+
+def exists(resource: Optional[ResourceInstance]) -> bool:
+    """Simple predicate to check for existence of a resource.
+
+    While a List type resource technically always exists, this will only return
+    true if the List contains items."""
+    return bool(resource) and not empty_list(resource)
+
+
+RESOURCE_PREDICATES = {
+    "DaemonSet": daemonset_ready,
+    "Deployment": deployment_ready,
+    "Pod": pod_ready,
+    "StatefulSet": statefulset_ready,
+}
+
+
+def empty_list(resource: ResourceInstance) -> bool:
+    return resource.kind.endswith("List") and not resource.get("items")
+
+
+def clock(total: int, interval: int) -> Iterator[int]:
+    start = time.monotonic()
+    yield 0
+    while (time.monotonic() - start) < total:
+        time.sleep(interval)
+        yield int(time.monotonic() - start)
+
+
+class Waiter:
+    def __init__(
+        self, client, resource: Resource, predicate: Callable[[ResourceInstance], bool]
+    ):
+        self.client = client
+        self.resource = resource
+        self.predicate = predicate
+
+    def wait(
+        self,
+        definition: Dict,
+        timeout: int,
+        sleep: int,
+        label_selectors: Optional[List[str]] = None,
+    ) -> Tuple[bool, Optional[Dict], int]:
+        params = {
+            "name": definition["metadata"].get("name"),
+            "namespace": definition["metadata"].get("namespace"),
+        }
+        if label_selectors:
+            params["label_selector"] = ",".join(label_selectors)
+        instance: Optional[Dict] = None
+        response = None
+        elapsed = 0
+        for i in clock(timeout, sleep):
+            elapsed = i
+            try:
+                response = self.client.get(self.resource, **params)
+            except NotFoundError:
+                pass
+            if self.predicate(response):
+                break
+        if response:
+            instance = response.to_dict()
+        return self.predicate(response), instance, elapsed
+
+
+class DummyWaiter:
+    """A no-op waiter that simply returns the item being waited on.
+
+    No API call will be made with this waiter; the function returns
+    immediately. This waiter is useful for waiting on resource instances in
+    check mode, for example.
+    """
+
+    def wait(
+        self,
+        definition: Dict,
+        timeout: int,
+        sleep: int,
+        label_selectors: Optional[List[str]] = None,
+    ) -> Tuple[bool, Optional[Dict], int]:
+        return True, definition, 0
+
+
+# The better solution would be typing.Protocol, but this is only in 3.8+
+SupportsWait = Union[Waiter, DummyWaiter]
+
+
+def get_waiter(
+    client,
+    resource: Resource,
+    state: str = "present",
+    condition: Optional[Dict] = None,
+    check_mode: Optional[bool] = False,
+) -> SupportsWait:
+    """Create a Waiter object based on the specified resource.
+
+    This is a convenience method for creating a waiter from a resource.
+    Based on the arguments and the kind of resource, an appropriate waiter
+    will be returned. A waiter can also be created directly, of course.
+    """
+    if check_mode:
+        return DummyWaiter()
+    if state == "present":
+        if condition:
+            predicate: Callable[[ResourceInstance], bool] = partial(
+                custom_condition, condition
+            )
+        else:
+            predicate = RESOURCE_PREDICATES.get(resource.kind, exists)
+    else:
+        predicate = resource_absent
+    return Waiter(client, resource, predicate)

tests/unit/module_utils/fixtures/definitions.yml

@@ -1,11 +1,34 @@
+---
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: test-1
+---
 kind: Pod
 apiVersion: v1
 metadata:
-  name: foo
-  namespace: bar
+  name: pod-1
+  namespace: test-1
+spec:
+  containers:
+    - image: busybox
+      name: busybox
 ---
-kind: ConfigMap
+kind: PodList
 apiVersion: v1
-metadata:
-  name: foo
-  namespace: bar
+metadata: {}
+items:
+  - kind: Pod
+    apiVersion: v1
+    metadata:
+      name: pod-1
+      namespace: test-1
+    spec:
+      containers:
+        - image: busybox
+          name: busybox
+---
+kind: ConfigMapList
+apiVersion: v1
+metadata: {}
+items: []

tests/unit/module_utils/fixtures/deployments.yml

@@ -0,0 +1,48 @@
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: deploy-1
+  namespace: test-1
+  generation: 1
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: foo
+  template:
+    metadata:
+      labels:
+        app: foo
+    spec:
+      containers:
+        - image: busybox
+          name: busybox
+status:
+  availableReplicas: 2
+  replicas: 2
+  observedGeneration: 1
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: deploy-2
+  namespace: test-1
+  generation: 1
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: foo
+  template:
+    metadata:
+      labels:
+        app: foo
+    spec:
+      containers:
+        - image: busybox
+          name: busybox
+status:
+  availableReplicas: 1
+  replicas: 2
+  observedGeneration: 1

tests/unit/module_utils/fixtures/pods.yml

@@ -0,0 +1,63 @@
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  namespace: test-1
+  name: pod-1
+spec:
+  containers:
+    - image: busybox
+      name: busybox
+status:
+  containerStatuses:
+    - name: busybox
+      ready: true
+  conditions:
+    - type: "www.example.com/gate"
+      status: "True"
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  namespace: test-1
+  name: pod-2
+spec:
+  containers:
+    - image: busybox
+      name: busybox
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  namespace: test-1
+  name: pod-3
+spec:
+  containers:
+    - image: busybox
+      name: busybox
+status:
+  phase: Pending
+  conditions:
+    - type: "www.example.com/gate"
+      status: "Unknown"
+  containerStatuses:
+    - name: busybox
+      ready: true
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  namespace: test-1
+  name: pod-4
+spec:
+  containers:
+    - image: busybox
+      name: busybox
+status:
+  phase: Pending
+  conditions:
+    - type: "www.example.com/other"
+      status: "Unknown"
+  containerStatuses:
+    - name: busybox
+      ready: true

tests/unit/module_utils/test_resource.py

@@ -30,9 +30,9 @@ def test_create_definitions_loads_from_file():
     current = Path(os.path.dirname(os.path.abspath(__file__)))
     params = {"src": current / "fixtures/definitions.yml"}
     results = create_definitions(params)
-    assert len(results) == 2
-    assert results[0].kind == "Pod"
-    assert results[1].kind == "ConfigMap"
+    assert len(results) == 3
+    assert results[0].kind == "Namespace"
+    assert results[1].kind == "Pod"
 
 
 def test_create_definitions_loads_from_params():
@@ -160,8 +160,8 @@ def test_from_yaml_loads_dictionary():
 def test_from_file_loads_definitions():
     current = Path(os.path.dirname(os.path.abspath(__file__)))
     result = list(from_file(current / "fixtures/definitions.yml"))
-    assert result[0]["kind"] == "Pod"
-    assert result[1]["kind"] == "ConfigMap"
+    assert result[0]["kind"] == "Namespace"
+    assert result[1]["kind"] == "Pod"
 
 
 def test_flatten_list_kind_flattens():

tests/unit/module_utils/test_waiter.py

@@ -0,0 +1,114 @@
+import os
+import time
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+import yaml
+from kubernetes.dynamic.resource import ResourceInstance
+from kubernetes.dynamic.exceptions import NotFoundError
+
+from ansible_collections.kubernetes.core.plugins.module_utils.k8s.waiter import (
+    clock,
+    custom_condition,
+    deployment_ready,
+    DummyWaiter,
+    exists,
+    get_waiter,
+    pod_ready,
+    resource_absent,
+    Waiter,
+)
+
+
+def resources(filepath):
+    current = Path(os.path.dirname(os.path.abspath(__file__)))
+    with open(current / filepath) as fp:
+        return [ResourceInstance(None, d) for d in yaml.safe_load_all(fp)]
+
+
+RESOURCES = resources("fixtures/definitions.yml")
+PODS = resources("fixtures/pods.yml")
+DEPLOYMENTS = resources("fixtures/deployments.yml")
+
+
+def test_clock_times_out():
+    start = time.monotonic()
+    for x in clock(5, 1):
+        pass
+    elapsed = int(time.monotonic() - start)
+    assert x == 5
+    assert 5 <= elapsed <= 6
+
+
+@pytest.mark.parametrize(
+    "resource,expected",
+    zip(RESOURCES + [None, {}], [True, True, True, False, False, False]),
+)
+def test_exists_and_absent_checks_for_existence(resource, expected):
+    assert exists(resource) is expected
+    assert resource_absent(resource) is not expected
+
+
+@pytest.mark.parametrize("pod,expected", zip(PODS, [True, False, True, True]))
+def test_pod_ready_checks_readiness(pod, expected):
+    assert pod_ready(pod) is expected
+
+
+@pytest.mark.parametrize("pod,expected", zip(PODS, [True, False, False, False]))
+def test_custom_condition_checks_readiness(pod, expected):
+    condition = {"type": "www.example.com/gate", "status": "True"}
+    assert custom_condition(condition, pod) is expected
+
+
+@pytest.mark.parametrize("deployment,expected", zip(DEPLOYMENTS, [True, False]))
+def test_deployment_ready_checks_readiness(deployment, expected):
+    assert deployment_ready(deployment) is expected
+
+
+def test_dummywaiter_returns_resource_immediately():
+    resource = {
+        "kind": "Pod",
+        "apiVersion": "v1",
+        "metadata": {"name": "foopod", "namespace": "foobar"},
+    }
+    result, instance, elapsed = DummyWaiter().wait(resource, 10, 100)
+    assert result is True
+    assert instance == resource
+    assert elapsed == 0
+
+
+def test_waiter_waits_for_missing_resource():
+    spec = {"get.side_effect": NotFoundError(Mock())}
+    client = Mock(**spec)
+    resource = Mock()
+    result, instance, elapsed = Waiter(client, resource, exists).wait(
+        RESOURCES[0], 3, 1
+    )
+    assert result is False
+    assert instance is None
+    assert abs(elapsed - 3) <= 1
+
+
+@pytest.mark.parametrize("resource,expected", zip(RESOURCES, [True, True, True, False]))
+def test_waiter_waits_for_resource_to_exist(resource, expected):
+    result = resource.to_dict()
+    spec = {"get.side_effect": [NotFoundError(Mock()), resource, resource, resource]}
+    client = Mock(**spec)
+    success, instance, elapsed = Waiter(client, Mock(), exists).wait(result, 3, 1)
+    assert success is expected
+    assert instance == result
+    assert abs(elapsed - 2) <= 1
+
+
+def test_get_waiter_returns_correct_waiter():
+    assert get_waiter(Mock(), PODS[0]).predicate == pod_ready
+    waiter = get_waiter(Mock(), PODS[0], check_mode=True)
+    assert isinstance(waiter, DummyWaiter)
+    assert get_waiter(Mock(), PODS[0], state="absent").predicate == resource_absent
+    assert (
+        get_waiter(
+            Mock(), PODS[0], condition={"type": "Ready", "status": "True"}
+        ).predicate.func
+        == custom_condition
+    )