From f168a3f67f4f2d4ffa5ae048d402f7b4e3098e49 Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Mon, 13 Dec 2021 11:06:13 -0500 Subject: [PATCH] Add new waiter (#306) Add new waiter SUMMARY This refactors the waiter logic from common.py into a separate module. ISSUE TYPE COMPONENT NAME ADDITIONAL INFORMATION Reviewed-by: None Reviewed-by: Alina Buzachis Reviewed-by: None --- plugins/module_utils/k8s/waiter.py | 204 ++++++++++++++++++ .../module_utils/fixtures/definitions.yml | 35 ++- .../module_utils/fixtures/deployments.yml | 48 +++++ tests/unit/module_utils/fixtures/pods.yml | 63 ++++++ tests/unit/module_utils/test_resource.py | 10 +- tests/unit/module_utils/test_waiter.py | 114 ++++++++++ 6 files changed, 463 insertions(+), 11 deletions(-) create mode 100644 plugins/module_utils/k8s/waiter.py create mode 100644 tests/unit/module_utils/fixtures/deployments.yml create mode 100644 tests/unit/module_utils/fixtures/pods.yml create mode 100644 tests/unit/module_utils/test_waiter.py diff --git a/plugins/module_utils/k8s/waiter.py b/plugins/module_utils/k8s/waiter.py new file mode 100644 index 00000000..c0cc41a6 --- /dev/null +++ b/plugins/module_utils/k8s/waiter.py @@ -0,0 +1,204 @@ +import time +from functools import partial +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union + +from ansible.module_utils.parsing.convert_bool import boolean + +try: + from kubernetes.dynamic.exceptions import NotFoundError + from kubernetes.dynamic.resource import Resource, ResourceField, ResourceInstance +except ImportError: + # These are defined only for the sake of Ansible's checked import requirement + Resource = Any # type: ignore + ResourceInstance = Any # type: ignore + pass + + +def deployment_ready(deployment: ResourceInstance) -> bool: + # FIXME: frustratingly bool(deployment.status) is True even if status is empty + # Furthermore deployment.status.availableReplicas == deployment.status.replicas == None if status is empty + # deployment.status.replicas is None is perfectly ok if desired replicas == 0 + # Scaling up means that we also need to check that we're not in a + # situation where status.replicas == status.availableReplicas + # but spec.replicas != status.replicas + return bool( + deployment.status + and deployment.spec.replicas == (deployment.status.replicas or 0) + and deployment.status.availableReplicas == deployment.status.replicas + and deployment.status.observedGeneration == deployment.metadata.generation + and not deployment.status.unavailableReplicas + ) + + +def pod_ready(pod: ResourceInstance) -> bool: + return bool( + pod.status + and pod.status.containerStatuses is not None + and all(container.ready for container in pod.status.containerStatuses) + ) + + +def daemonset_ready(daemonset: ResourceInstance) -> bool: + return bool( + daemonset.status + and daemonset.status.desiredNumberScheduled is not None + and daemonset.status.updatedNumberScheduled + == daemonset.status.desiredNumberScheduled + and daemonset.status.numberReady == daemonset.status.desiredNumberScheduled + and daemonset.status.observedGeneration == daemonset.metadata.generation + and not daemonset.status.unavailableReplicas + ) + + +def statefulset_ready(statefulset: ResourceInstance) -> bool: + return bool( + statefulset.status + and statefulset.spec.updateStrategy.type == "RollingUpdate" + and statefulset.status.observedGeneration + == (statefulset.metadata.generation or 0) + and statefulset.status.updateRevision == statefulset.status.currentRevision + and statefulset.status.updatedReplicas == statefulset.spec.replicas + and statefulset.status.readyReplicas == statefulset.spec.replicas + and statefulset.status.replicas == statefulset.spec.replicas + ) + + +def custom_condition(condition: Dict, resource: ResourceInstance) -> bool: + if not resource.status or not resource.status.conditions: + return False + matches = [x for x in resource.status.conditions if x.type == condition["type"]] + if not matches: + return False + # There should never be more than one condition of a specific type + match: ResourceField = matches[0] + if match.status == "Unknown": + if match.status == condition["status"]: + if "reason" not in condition: + return True + if condition["reason"]: + return match.reason == condition["reason"] + return False + status = True if match.status == "True" else False + if status == boolean(condition["status"], strict=False): + if condition.get("reason"): + return match.reason == condition["reason"] + return True + return False + + +def resource_absent(resource: ResourceInstance) -> bool: + return not exists(resource) + + +def exists(resource: Optional[ResourceInstance]) -> bool: + """Simple predicate to check for existence of a resource. + + While a List type resource technically always exists, this will only return + true if the List contains items.""" + return bool(resource) and not empty_list(resource) + + +RESOURCE_PREDICATES = { + "DaemonSet": daemonset_ready, + "Deployment": deployment_ready, + "Pod": pod_ready, + "StatefulSet": statefulset_ready, +} + + +def empty_list(resource: ResourceInstance) -> bool: + return resource.kind.endswith("List") and not resource.get("items") + + +def clock(total: int, interval: int) -> Iterator[int]: + start = time.monotonic() + yield 0 + while (time.monotonic() - start) < total: + time.sleep(interval) + yield int(time.monotonic() - start) + + +class Waiter: + def __init__( + self, client, resource: Resource, predicate: Callable[[ResourceInstance], bool] + ): + self.client = client + self.resource = resource + self.predicate = predicate + + def wait( + self, + definition: Dict, + timeout: int, + sleep: int, + label_selectors: Optional[List[str]] = None, + ) -> Tuple[bool, Optional[Dict], int]: + params = { + "name": definition["metadata"].get("name"), + "namespace": definition["metadata"].get("namespace"), + } + if label_selectors: + params["label_selector"] = ",".join(label_selectors) + instance: Optional[Dict] = None + response = None + elapsed = 0 + for i in clock(timeout, sleep): + elapsed = i + try: + response = self.client.get(self.resource, **params) + except NotFoundError: + pass + if self.predicate(response): + break + if response: + instance = response.to_dict() + return self.predicate(response), instance, elapsed + + +class DummyWaiter: + """A no-op waiter that simply returns the item being waited on. + + No API call will be made with this waiter; the function returns + immediately. This waiter is useful for waiting on resource instances in + check mode, for example. + """ + + def wait( + self, + definition: Dict, + timeout: int, + sleep: int, + label_selectors: Optional[List[str]] = None, + ) -> Tuple[bool, Optional[Dict], int]: + return True, definition, 0 + + +# The better solution would be typing.Protocol, but this is only in 3.8+ +SupportsWait = Union[Waiter, DummyWaiter] + + +def get_waiter( + client, + resource: Resource, + state: str = "present", + condition: Optional[Dict] = None, + check_mode: Optional[bool] = False, +) -> SupportsWait: + """Create a Waiter object based on the specified resource. + + This is a convenience method for creating a waiter from a resource. + Based on the arguments and the kind of resource, an appropriate waiter + will be returned. A waiter can also be created directly, of course. + """ + if check_mode: + return DummyWaiter() + if state == "present": + if condition: + predicate: Callable[[ResourceInstance], bool] = partial( + custom_condition, condition + ) + else: + predicate = RESOURCE_PREDICATES.get(resource.kind, exists) + else: + predicate = resource_absent + return Waiter(client, resource, predicate) diff --git a/tests/unit/module_utils/fixtures/definitions.yml b/tests/unit/module_utils/fixtures/definitions.yml index d627bd5c..a8f6de81 100644 --- a/tests/unit/module_utils/fixtures/definitions.yml +++ b/tests/unit/module_utils/fixtures/definitions.yml @@ -1,11 +1,34 @@ +--- +kind: Namespace +apiVersion: v1 +metadata: + name: test-1 +--- kind: Pod apiVersion: v1 metadata: - name: foo - namespace: bar + name: pod-1 + namespace: test-1 +spec: + containers: + - image: busybox + name: busybox --- -kind: ConfigMap +kind: PodList apiVersion: v1 -metadata: - name: foo - namespace: bar +metadata: {} +items: + - kind: Pod + apiVersion: v1 + metadata: + name: pod-1 + namespace: test-1 + spec: + containers: + - image: busybox + name: busybox +--- +kind: ConfigMapList +apiVersion: v1 +metadata: {} +items: [] diff --git a/tests/unit/module_utils/fixtures/deployments.yml b/tests/unit/module_utils/fixtures/deployments.yml new file mode 100644 index 00000000..530035fc --- /dev/null +++ b/tests/unit/module_utils/fixtures/deployments.yml @@ -0,0 +1,48 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: deploy-1 + namespace: test-1 + generation: 1 +spec: + replicas: 2 + selector: + matchLabels: + app: foo + template: + metadata: + labels: + app: foo + spec: + containers: + - image: busybox + name: busybox +status: + availableReplicas: 2 + replicas: 2 + observedGeneration: 1 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: deploy-2 + namespace: test-1 + generation: 1 +spec: + replicas: 2 + selector: + matchLabels: + app: foo + template: + metadata: + labels: + app: foo + spec: + containers: + - image: busybox + name: busybox +status: + availableReplicas: 1 + replicas: 2 + observedGeneration: 1 diff --git a/tests/unit/module_utils/fixtures/pods.yml b/tests/unit/module_utils/fixtures/pods.yml new file mode 100644 index 00000000..250354e2 --- /dev/null +++ b/tests/unit/module_utils/fixtures/pods.yml @@ -0,0 +1,63 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + namespace: test-1 + name: pod-1 +spec: + containers: + - image: busybox + name: busybox +status: + containerStatuses: + - name: busybox + ready: true + conditions: + - type: "www.example.com/gate" + status: "True" +--- +kind: Pod +apiVersion: v1 +metadata: + namespace: test-1 + name: pod-2 +spec: + containers: + - image: busybox + name: busybox +--- +kind: Pod +apiVersion: v1 +metadata: + namespace: test-1 + name: pod-3 +spec: + containers: + - image: busybox + name: busybox +status: + phase: Pending + conditions: + - type: "www.example.com/gate" + status: "Unknown" + containerStatuses: + - name: busybox + ready: true +--- +kind: Pod +apiVersion: v1 +metadata: + namespace: test-1 + name: pod-4 +spec: + containers: + - image: busybox + name: busybox +status: + phase: Pending + conditions: + - type: "www.example.com/other" + status: "Unknown" + containerStatuses: + - name: busybox + ready: true diff --git a/tests/unit/module_utils/test_resource.py b/tests/unit/module_utils/test_resource.py index 8d7beda1..c27c01a8 100644 --- a/tests/unit/module_utils/test_resource.py +++ b/tests/unit/module_utils/test_resource.py @@ -30,9 +30,9 @@ def test_create_definitions_loads_from_file(): current = Path(os.path.dirname(os.path.abspath(__file__))) params = {"src": current / "fixtures/definitions.yml"} results = create_definitions(params) - assert len(results) == 2 - assert results[0].kind == "Pod" - assert results[1].kind == "ConfigMap" + assert len(results) == 3 + assert results[0].kind == "Namespace" + assert results[1].kind == "Pod" def test_create_definitions_loads_from_params(): @@ -160,8 +160,8 @@ def test_from_yaml_loads_dictionary(): def test_from_file_loads_definitions(): current = Path(os.path.dirname(os.path.abspath(__file__))) result = list(from_file(current / "fixtures/definitions.yml")) - assert result[0]["kind"] == "Pod" - assert result[1]["kind"] == "ConfigMap" + assert result[0]["kind"] == "Namespace" + assert result[1]["kind"] == "Pod" def test_flatten_list_kind_flattens(): diff --git a/tests/unit/module_utils/test_waiter.py b/tests/unit/module_utils/test_waiter.py new file mode 100644 index 00000000..5ed76743 --- /dev/null +++ b/tests/unit/module_utils/test_waiter.py @@ -0,0 +1,114 @@ +import os +import time +from pathlib import Path +from unittest.mock import Mock + +import pytest +import yaml +from kubernetes.dynamic.resource import ResourceInstance +from kubernetes.dynamic.exceptions import NotFoundError + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.waiter import ( + clock, + custom_condition, + deployment_ready, + DummyWaiter, + exists, + get_waiter, + pod_ready, + resource_absent, + Waiter, +) + + +def resources(filepath): + current = Path(os.path.dirname(os.path.abspath(__file__))) + with open(current / filepath) as fp: + return [ResourceInstance(None, d) for d in yaml.safe_load_all(fp)] + + +RESOURCES = resources("fixtures/definitions.yml") +PODS = resources("fixtures/pods.yml") +DEPLOYMENTS = resources("fixtures/deployments.yml") + + +def test_clock_times_out(): + start = time.monotonic() + for x in clock(5, 1): + pass + elapsed = int(time.monotonic() - start) + assert x == 5 + assert 5 <= elapsed <= 6 + + +@pytest.mark.parametrize( + "resource,expected", + zip(RESOURCES + [None, {}], [True, True, True, False, False, False]), +) +def test_exists_and_absent_checks_for_existence(resource, expected): + assert exists(resource) is expected + assert resource_absent(resource) is not expected + + +@pytest.mark.parametrize("pod,expected", zip(PODS, [True, False, True, True])) +def test_pod_ready_checks_readiness(pod, expected): + assert pod_ready(pod) is expected + + +@pytest.mark.parametrize("pod,expected", zip(PODS, [True, False, False, False])) +def test_custom_condition_checks_readiness(pod, expected): + condition = {"type": "www.example.com/gate", "status": "True"} + assert custom_condition(condition, pod) is expected + + +@pytest.mark.parametrize("deployment,expected", zip(DEPLOYMENTS, [True, False])) +def test_deployment_ready_checks_readiness(deployment, expected): + assert deployment_ready(deployment) is expected + + +def test_dummywaiter_returns_resource_immediately(): + resource = { + "kind": "Pod", + "apiVersion": "v1", + "metadata": {"name": "foopod", "namespace": "foobar"}, + } + result, instance, elapsed = DummyWaiter().wait(resource, 10, 100) + assert result is True + assert instance == resource + assert elapsed == 0 + + +def test_waiter_waits_for_missing_resource(): + spec = {"get.side_effect": NotFoundError(Mock())} + client = Mock(**spec) + resource = Mock() + result, instance, elapsed = Waiter(client, resource, exists).wait( + RESOURCES[0], 3, 1 + ) + assert result is False + assert instance is None + assert abs(elapsed - 3) <= 1 + + +@pytest.mark.parametrize("resource,expected", zip(RESOURCES, [True, True, True, False])) +def test_waiter_waits_for_resource_to_exist(resource, expected): + result = resource.to_dict() + spec = {"get.side_effect": [NotFoundError(Mock()), resource, resource, resource]} + client = Mock(**spec) + success, instance, elapsed = Waiter(client, Mock(), exists).wait(result, 3, 1) + assert success is expected + assert instance == result + assert abs(elapsed - 2) <= 1 + + +def test_get_waiter_returns_correct_waiter(): + assert get_waiter(Mock(), PODS[0]).predicate == pod_ready + waiter = get_waiter(Mock(), PODS[0], check_mode=True) + assert isinstance(waiter, DummyWaiter) + assert get_waiter(Mock(), PODS[0], state="absent").predicate == resource_absent + assert ( + get_waiter( + Mock(), PODS[0], condition={"type": "Ready", "status": "True"} + ).predicate.func + == custom_condition + )