Replicate apply method in the DynamicClient (#45)

* * Replicate apply method in the DynamicClient

Signed-off-by: Alina Buzachis <abuzachis@redhat.com>

* * Add changelog fragment

Signed-off-by: Alina Buzachis <abuzachis@redhat.com>

changelogs/fragments/45-add-apply-method.yml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+- Replicate apply method in the DynamicClient (https://github.com/ansible-collections/kubernetes.core/pull/45).

plugins/module_utils/apply.py

@@ -0,0 +1,290 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from collections import OrderedDict
+import json
+import sys
+
+from ansible.module_utils.common.dict_transformations import dict_merge
+from ansible.module_utils.six import PY3
+from ansible_collections.kubernetes.core.plugins.module_utils.exceptions import ApplyException
+
+try:
+    from kubernetes.dynamic.exceptions import NotFoundError
+except ImportError:
+    pass
+
+if PY3:
+    unicode = str
+
+LAST_APPLIED_CONFIG_ANNOTATION = 'kubectl.kubernetes.io/last-applied-configuration'
+
+POD_SPEC_SUFFIXES = {
+    'containers': 'name',
+    'initContainers': 'name',
+    'ephemeralContainers': 'name',
+    'volumes': 'name',
+    'imagePullSecrets': 'name',
+    'containers.volumeMounts': 'mountPath',
+    'containers.volumeDevices': 'devicePath',
+    'containers.env': 'name',
+    'containers.ports': 'containerPort',
+    'initContainers.volumeMounts': 'mountPath',
+    'initContainers.volumeDevices': 'devicePath',
+    'initContainers.env': 'name',
+    'initContainers.ports': 'containerPort',
+    'ephemeralContainers.volumeMounts': 'mountPath',
+    'ephemeralContainers.volumeDevices': 'devicePath',
+    'ephemeralContainers.env': 'name',
+    'ephemeralContainers.ports': 'containerPort',
+}
+
+POD_SPEC_PREFIXES = [
+    'Pod.spec',
+    'Deployment.spec.template.spec',
+    'DaemonSet.spec.template.spec',
+    'StatefulSet.spec.template.spec',
+    'Job.spec.template.spec',
+    'Cronjob.spec.jobTemplate.spec.template.spec',
+]
+
+# patch merge keys taken from generated.proto files under
+# staging/src/k8s.io/api in kubernetes/kubernetes
+STRATEGIC_MERGE_PATCH_KEYS = {
+    'Service.spec.ports': 'port',
+    'ServiceAccount.secrets': 'name',
+    'ValidatingWebhookConfiguration.webhooks': 'name',
+    'MutatingWebhookConfiguration.webhooks': 'name',
+}
+
+STRATEGIC_MERGE_PATCH_KEYS.update(
+    {"%s.%s" % (prefix, key): value
+     for prefix in POD_SPEC_PREFIXES
+     for key, value in POD_SPEC_SUFFIXES.items()}
+)
+
+
+if sys.version_info.major >= 3:
+    json_loads_byteified = json.loads
+else:
+    # https://stackoverflow.com/a/33571117
+    def json_loads_byteified(json_text):
+        return _byteify(
+            json.loads(json_text, object_hook=_byteify),
+            ignore_dicts=True
+        )
+
+    def _byteify(data, ignore_dicts=False):
+        # if this is a unicode string, return its string representation
+        if isinstance(data, unicode):  # noqa: F821
+            return data.encode('utf-8')
+        # if this is a list of values, return list of byteified values
+        if isinstance(data, list):
+            return [_byteify(item, ignore_dicts=True) for item in data]
+        # if this is a dictionary, return dictionary of byteified keys and values
+        # but only if we haven't already byteified it
+        if isinstance(data, dict) and not ignore_dicts:
+            return {
+                _byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)
+                for key, value in data.items()
+            }
+        # if it's anything else, return it in its original form
+        return data
+
+
+def annotate(desired):
+    return dict(
+        metadata=dict(
+            annotations={
+                LAST_APPLIED_CONFIG_ANNOTATION: json.dumps(desired, separators=(',', ':'), indent=None, sort_keys=True)
+            }
+        )
+    )
+
+
+def apply_patch(actual, desired):
+    last_applied = actual['metadata'].get('annotations', {}).get(LAST_APPLIED_CONFIG_ANNOTATION)
+
+    if last_applied:
+        # ensure that last_applied doesn't come back as a dict of unicode key/value pairs
+        # json.loads can be used if we stop supporting python 2
+        last_applied = json_loads_byteified(last_applied)
+        patch = merge(dict_merge(last_applied, annotate(last_applied)),
+                      dict_merge(desired, annotate(desired)), actual)
+        if patch:
+            return actual, patch
+        else:
+            return actual, actual
+    else:
+        return actual, dict_merge(desired, annotate(desired))
+
+
+def apply_object(resource, definition):
+    try:
+        actual = resource.get(name=definition['metadata']['name'], namespace=definition['metadata'].get('namespace'))
+    except NotFoundError:
+        return None, dict_merge(definition, annotate(definition))
+    return apply_patch(actual.to_dict(), definition)
+
+
+def apply(resource, definition):
+    existing, desired = apply_object(resource, definition)
+    if not existing:
+        return resource.create(body=desired, namespace=definition['metadata'].get('namespace'))
+    if existing == desired:
+        return resource.get(name=definition['metadata']['name'], namespace=definition['metadata'].get('namespace'))
+    return resource.patch(body=desired,
+                          name=definition['metadata']['name'],
+                          namespace=definition['metadata'].get('namespace'),
+                          content_type='application/merge-patch+json')
+
+
+# The patch is the difference from actual to desired without deletions, plus deletions
+# from last_applied to desired. To find it, we compute deletions, which are the deletions from
+# last_applied to desired, and delta, which is the difference from actual to desired without
+# deletions, and then apply delta to deletions as a patch, which should be strictly additive.
+def merge(last_applied, desired, actual, position=None):
+    deletions = get_deletions(last_applied, desired)
+    delta = get_delta(last_applied, actual, desired, position or desired['kind'])
+    return dict_merge(deletions, delta)
+
+
+def list_to_dict(lst, key, position):
+    result = OrderedDict()
+    for item in lst:
+        try:
+            result[item[key]] = item
+        except KeyError:
+            raise ApplyException("Expected key '%s' not found in position %s" % (key, position))
+    return result
+
+
+# list_merge applies a strategic merge to a set of lists if the patchMergeKey is known
+# each item in the list is compared based on the patchMergeKey - if two values with the
+# same patchMergeKey differ, we take the keys that are in last applied, compare the
+# actual and desired for those keys, and update if any differ
+def list_merge(last_applied, actual, desired, position):
+    result = list()
+    if position in STRATEGIC_MERGE_PATCH_KEYS and last_applied:
+        patch_merge_key = STRATEGIC_MERGE_PATCH_KEYS[position]
+        last_applied_dict = list_to_dict(last_applied, patch_merge_key, position)
+        actual_dict = list_to_dict(actual, patch_merge_key, position)
+        desired_dict = list_to_dict(desired, patch_merge_key, position)
+        for key in desired_dict:
+            if key not in actual_dict or key not in last_applied_dict:
+                result.append(desired_dict[key])
+            else:
+                patch = merge(last_applied_dict[key], desired_dict[key], actual_dict[key], position)
+                result.append(dict_merge(actual_dict[key], patch))
+        for key in actual_dict:
+            if key not in desired_dict and key not in last_applied_dict:
+                result.append(actual_dict[key])
+        return result
+    else:
+        return desired
+
+
+def recursive_list_diff(list1, list2, position=None):
+    result = (list(), list())
+    if position in STRATEGIC_MERGE_PATCH_KEYS:
+        patch_merge_key = STRATEGIC_MERGE_PATCH_KEYS[position]
+        dict1 = list_to_dict(list1, patch_merge_key, position)
+        dict2 = list_to_dict(list2, patch_merge_key, position)
+        dict1_keys = set(dict1.keys())
+        dict2_keys = set(dict2.keys())
+        for key in dict1_keys - dict2_keys:
+            result[0].append(dict1[key])
+        for key in dict2_keys - dict1_keys:
+            result[1].append(dict2[key])
+        for key in dict1_keys & dict2_keys:
+            diff = recursive_diff(dict1[key], dict2[key], position)
+            if diff:
+                # reinsert patch merge key to relate changes in other keys to
+                # a specific list element
+                diff[0].update({patch_merge_key: dict1[key][patch_merge_key]})
+                diff[1].update({patch_merge_key: dict2[key][patch_merge_key]})
+                result[0].append(diff[0])
+                result[1].append(diff[1])
+        if result[0] or result[1]:
+            return result
+    elif list1 != list2:
+        return (list1, list2)
+    return None
+
+
+def recursive_diff(dict1, dict2, position=None):
+    if not position:
+        if 'kind' in dict1 and dict1.get('kind') == dict2.get('kind'):
+            position = dict1['kind']
+    left = dict((k, v) for (k, v) in dict1.items() if k not in dict2)
+    right = dict((k, v) for (k, v) in dict2.items() if k not in dict1)
+    for k in (set(dict1.keys()) & set(dict2.keys())):
+        if position:
+            this_position = "%s.%s" % (position, k)
+        if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
+            result = recursive_diff(dict1[k], dict2[k], this_position)
+            if result:
+                left[k] = result[0]
+                right[k] = result[1]
+        elif isinstance(dict1[k], list) and isinstance(dict2[k], list):
+            result = recursive_list_diff(dict1[k], dict2[k], this_position)
+            if result:
+                left[k] = result[0]
+                right[k] = result[1]
+        elif dict1[k] != dict2[k]:
+            left[k] = dict1[k]
+            right[k] = dict2[k]
+    if left or right:
+        return left, right
+    else:
+        return None
+
+
+def get_deletions(last_applied, desired):
+    patch = {}
+    for k, last_applied_value in last_applied.items():
+        desired_value = desired.get(k)
+        if isinstance(last_applied_value, dict) and isinstance(desired_value, dict):
+            p = get_deletions(last_applied_value, desired_value)
+            if p:
+                patch[k] = p
+        elif last_applied_value != desired_value:
+            patch[k] = desired_value
+    return patch
+
+
+def get_delta(last_applied, actual, desired, position=None):
+    patch = {}
+
+    for k, desired_value in desired.items():
+        if position:
+            this_position = "%s.%s" % (position, k)
+        actual_value = actual.get(k)
+        if actual_value is None:
+            patch[k] = desired_value
+        elif isinstance(desired_value, dict):
+            p = get_delta(last_applied.get(k, {}), actual_value, desired_value, this_position)
+            if p:
+                patch[k] = p
+        elif isinstance(desired_value, list):
+            p = list_merge(last_applied.get(k, []), actual_value, desired_value, this_position)
+            if p:
+                patch[k] = [item for item in p if item]
+        elif actual_value != desired_value:
+            patch[k] = desired_value
+    return patch

plugins/module_utils/common.py

@@ -43,10 +43,10 @@ K8S_IMP_ERR = None
 try:
     import kubernetes
     import openshift
-    from openshift.dynamic import DynamicClient
-    from openshift.dynamic.exceptions import (
-        ResourceNotFoundError, ResourceNotUniqueError, NotFoundError, DynamicApiError,
-        ConflictError, ForbiddenError, MethodNotAllowedError)
+    from kubernetes.dynamic.exceptions import (
+        NotFoundError, ResourceNotFoundError, ResourceNotUniqueError, DynamicApiError,
+        ConflictError, ForbiddenError, MethodNotAllowedError, BadRequestError
+    )
     HAS_K8S_MODULE_HELPER = True
     k8s_import_exception = None
 except ImportError as e:
@@ -54,6 +54,15 @@ except ImportError as e:
     k8s_import_exception = e
     K8S_IMP_ERR = traceback.format_exc()
 
+IMP_K8S_CLIENT = None
+try:
+    from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import K8SDynamicClient
+    IMP_K8S_CLIENT = True
+except ImportError as e:
+    IMP_K8S_CLIENT = False
+    k8s_client_import_exception = e
+    IMP_K8S_CLIENT_ERR = traceback.format_exc()
+
 YAML_IMP_ERR = None
 try:
     import yaml
@@ -64,7 +73,7 @@ except ImportError:
 
 K8S_CONFIG_HASH_IMP_ERR = None
 try:
-    from openshift.dynamic.exceptions import KubernetesValidateMissing
+    from kubernetes.dynamic.exceptions import KubernetesValidateMissing
     HAS_K8S_CONFIG_HASH = True
 except ImportError:
     K8S_CONFIG_HASH_IMP_ERR = traceback.format_exc()
@@ -72,7 +81,7 @@ except ImportError:
 
 HAS_K8S_APPLY = None
 try:
-    from openshift.dynamic.apply import apply_object
+    from ansible_collections.kubernetes.core.plugins.module_utils.apply import apply_object
     HAS_K8S_APPLY = True
 except ImportError:
     HAS_K8S_APPLY = False
@@ -84,17 +93,12 @@ except ImportError:
     pass
 
 try:
-    from openshift.dynamic.apply import recursive_diff
+    from ansible_collections.kubernetes.core.plugins.module_utils.apply import recursive_diff
 except ImportError:
     from ansible.module_utils.common.dict_transformations import recursive_diff
 
 try:
-    try:
-        # >=0.10
-        from openshift.dynamic.resource import ResourceInstance
-    except ImportError:
-        # <0.10
-        from openshift.dynamic.client import ResourceInstance
+    from kubernetes.dynamic.resource import ResourceInstance
     HAS_K8S_INSTANCE_HELPER = True
     k8s_import_exception = None
 except ImportError as e:
@@ -194,7 +198,7 @@ def get_api_client(module=None, **kwargs):
     cache_file = generate_cache_file(kubeclient)
 
     try:
-        client = DynamicClient(kubeclient, cache_file)
+        client = K8SDynamicClient(kubeclient, cache_file)
     except Exception as err:
         _raise_or_fail(err, 'Failed to get client due to %s')
 
@@ -245,9 +249,9 @@ class K8sAnsibleMixin(object):
             result = resource.get(name=name, namespace=namespace,
                                   label_selector=','.join(label_selectors),
                                   field_selector=','.join(field_selectors))
-        except openshift.dynamic.exceptions.BadRequestError:
+        except BadRequestError:
             return dict(resources=[], api_found=True)
-        except openshift.dynamic.exceptions.NotFoundError:
+        except NotFoundError:
             if not wait or name is None:
                 return dict(resources=[], api_found=True)
 

plugins/module_utils/exceptions.py

@@ -0,0 +1,21 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+class ApplyException(Exception):
+    """ Could not apply patch """

plugins/module_utils/k8sdynamicclient.py

@@ -0,0 +1,39 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+from kubernetes.dynamic import DynamicClient
+
+from ansible_collections.kubernetes.core.plugins.module_utils.apply import apply
+from ansible_collections.kubernetes.core.plugins.module_utils.exceptions import ApplyException
+
+
+class K8SDynamicClient(DynamicClient):
+    def apply(self, resource, body=None, name=None, namespace=None):
+        body = super().serialize_body(body)
+        body['metadata'] = body.get('metadata', dict())
+        name = name or body['metadata'].get('name')
+        if not name:
+            raise ValueError("name is required to apply {0}.{1}".format(resource.group_version, resource.kind))
+        if resource.namespaced:
+            body['metadata']['namespace'] = super().ensure_namespace(resource, namespace, body)
+        try:
+            return apply(resource, body)
+        except ApplyException as e:
+            raise ValueError("Could not apply strategic merge to %s/%s: %s" %
+                             (body['kind'], body['metadata']['name'], e))

tests/sanity/ignore-2.10.txt

@@ -8,3 +8,5 @@ molecule/default/roles/helm/files/appversionless-chart-v2/templates/configmap.ya
 molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
+plugins/module_utils/k8sdynamicclient.py import-2.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip

tests/sanity/ignore-2.11.txt

@@ -8,3 +8,5 @@ molecule/default/roles/helm/files/appversionless-chart-v2/templates/configmap.ya
 molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
+plugins/module_utils/k8sdynamicclient.py import-2.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip

tests/sanity/ignore-2.12.txt

@@ -8,3 +8,5 @@ molecule/default/roles/helm/files/appversionless-chart-v2/templates/configmap.ya
 molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
+plugins/module_utils/k8sdynamicclient.py import-2.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip

tests/sanity/ignore-2.9.txt

@@ -5,3 +5,5 @@ molecule/default/roles/helm/files/appversionless-chart-v2/templates/configmap.ya
 molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
+plugins/module_utils/k8sdynamicclient.py import-2.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip

tests/unit/module_utils/test_apply.py

@@ -0,0 +1,421 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible_collections.kubernetes.core.plugins.module_utils.apply import merge, apply_patch
+
+tests = [
+    dict(
+        last_applied=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo"),
+            data=dict(one="1", two="2")
+        ),
+        desired=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo"),
+            data=dict(one="1", two="2")
+        ),
+        expected={}
+    ),
+    dict(
+        last_applied=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo"),
+            data=dict(one="1", two="2")
+        ),
+        desired=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo"),
+            data=dict(one="1", two="2", three="3")
+        ),
+        expected=dict(data=dict(three="3"))
+    ),
+    dict(
+        last_applied=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo"),
+            data=dict(one="1", two="2")
+        ),
+        desired=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo"),
+            data=dict(one="1", three="3")
+        ),
+        expected=dict(data=dict(two=None, three="3"))
+    ),
+    dict(
+        last_applied=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo", annotations=dict(this="one", hello="world")),
+            data=dict(one="1", two="2")
+        ),
+        desired=dict(
+            kind="ConfigMap",
+            metadata=dict(name="foo"),
+            data=dict(one="1", three="3")
+        ),
+        expected=dict(metadata=dict(annotations=None), data=dict(two=None, three="3"))
+    ),
+
+    dict(
+        last_applied=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, name="http")])
+        ),
+        actual=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, protocol='TCP', name="http")])
+        ),
+        desired=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, name="http")])
+        ),
+        expected=dict(spec=dict(ports=[dict(port=8080, protocol='TCP', name="http")]))
+    ),
+    dict(
+        last_applied=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, name="http")])
+        ),
+        actual=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, protocol='TCP', name="http")])
+        ),
+        desired=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8081, name="http")])
+        ),
+        expected=dict(spec=dict(ports=[dict(port=8081, name="http")]))
+    ),
+    dict(
+        last_applied=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, name="http")])
+        ),
+        actual=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, protocol='TCP', name="http")])
+        ),
+        desired=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8443, name="https"), dict(port=8080, name="http")])
+        ),
+        expected=dict(spec=dict(ports=[dict(port=8443, name="https"), dict(port=8080, name="http", protocol='TCP')]))
+    ),
+    dict(
+        last_applied=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8443, name="https"), dict(port=8080, name="http")])
+        ),
+        actual=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8443, protocol='TCP', name="https"), dict(port=8080, protocol='TCP', name='http')])
+        ),
+        desired=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8080, name="http")])
+        ),
+        expected=dict(spec=dict(ports=[dict(port=8080, name="http", protocol='TCP')]))
+    ),
+    dict(
+        last_applied=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8443, name="https", madeup="xyz"), dict(port=8080, name="http")])
+        ),
+        actual=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8443, protocol='TCP', name="https", madeup="xyz"), dict(port=8080, protocol='TCP', name='http')])
+        ),
+        desired=dict(
+            kind="Service",
+            metadata=dict(name="foo"),
+            spec=dict(ports=[dict(port=8443, name="https")])
+        ),
+        expected=dict(spec=dict(ports=[dict(madeup=None, port=8443, name="https", protocol='TCP')]))
+    ),
+    dict(
+        last_applied=dict(
+            kind="Pod",
+            metadata=dict(name="foo"),
+            spec=dict(containers=[dict(name="busybox", image="busybox",
+                                       resources=dict(requests=dict(cpu="100m", memory="100Mi"), limits=dict(cpu="100m", memory="100Mi")))])
+        ),
+        actual=dict(
+            kind="Pod",
+            metadata=dict(name="foo"),
+            spec=dict(containers=[dict(name="busybox", image="busybox",
+                                       resources=dict(requests=dict(cpu="100m", memory="100Mi"), limits=dict(cpu="100m", memory="100Mi")))])
+        ),
+        desired=dict(
+            kind="Pod",
+            metadata=dict(name="foo"),
+            spec=dict(containers=[dict(name="busybox", image="busybox",
+                                       resources=dict(requests=dict(cpu="50m", memory="50Mi"), limits=dict(memory="50Mi")))])
+        ),
+        expected=dict(spec=dict(containers=[dict(name="busybox", image="busybox",
+                                                 resources=dict(requests=dict(cpu="50m", memory="50Mi"), limits=dict(cpu=None, memory="50Mi")))]))
+    ),
+    dict(
+        desired=dict(kind='Pod',
+                     spec=dict(containers=[
+                         dict(name='hello',
+                              volumeMounts=[dict(name="test", mountPath="/test")])
+                     ],
+                         volumes=[
+                             dict(name="test", configMap=dict(name="test")),
+                     ])),
+        last_applied=dict(kind='Pod',
+                          spec=dict(containers=[
+                              dict(name='hello',
+                                   volumeMounts=[dict(name="test", mountPath="/test")])
+                          ],
+                              volumes=[
+                                  dict(name="test", configMap=dict(name="test"))])),
+        actual=dict(kind='Pod',
+                    spec=dict(containers=[
+                        dict(name='hello',
+                             volumeMounts=[
+                                 dict(name="test", mountPath="/test"),
+                                 dict(mountPath="/var/run/secrets/kubernetes.io/serviceaccount", name="default-token-xyz")])
+                    ],
+                        volumes=[
+                            dict(name="test", configMap=dict(name="test")),
+                            dict(name="default-token-xyz", secret=dict(secretName="default-token-xyz")),
+                    ])),
+        expected=dict(spec=dict(containers=[dict(name='hello',
+                                volumeMounts=[dict(name="test", mountPath="/test"),
+                                              dict(mountPath="/var/run/secrets/kubernetes.io/serviceaccount", name="default-token-xyz")])],
+                                volumes=[dict(name="test", configMap=dict(name="test")),
+                                         dict(name="default-token-xyz", secret=dict(secretName="default-token-xyz"))])),
+    ),
+
+    # This next one is based on a real world case where definition was mostly
+    # str type and everything else was mostly unicode type (don't ask me how)
+    dict(
+        last_applied={
+            u'kind': u'ConfigMap',
+            u'data': {u'one': '1', 'three': '3', 'two': '2'},
+            u'apiVersion': u'v1',
+            u'metadata': {u'namespace': u'apply', u'name': u'apply-configmap'}
+        },
+        actual={
+            u'kind': u'ConfigMap',
+            u'data': {u'one': '1', 'three': '3', 'two': '2'},
+            u'apiVersion': u'v1',
+            u'metadata': {u'namespace': u'apply', u'name': u'apply-configmap',
+                          u'resourceVersion': '1714994',
+                          u'creationTimestamp': u'2019-08-17T05:08:05Z', u'annotations': {},
+                          u'selfLink': u'/api/v1/namespaces/apply/configmaps/apply-configmap',
+                          u'uid': u'fed45fb0-c0ac-11e9-9d95-025000000001'}
+        },
+        desired={
+            'kind': u'ConfigMap',
+            'data': {'one': '1', 'three': '3', 'two': '2'},
+            'apiVersion': 'v1',
+            'metadata': {'namespace': 'apply', 'name': 'apply-configmap'}
+        },
+        expected=dict()
+    ),
+    # apply a Deployment, then scale the Deployment (which doesn't affect last-applied)
+    # then apply the Deployment again. Should un-scale the Deployment
+    dict(
+        last_applied={
+            'kind': u'Deployment',
+            'spec': {
+                'replicas': 1,
+                'template': {
+                    'spec': {
+                        'containers': [
+                            {
+                                'name': 'this_must_exist',
+                                'envFrom': [
+                                    {
+                                        'configMapRef': {
+                                            'name': 'config-xyz'
+                                        }
+                                    },
+                                    {
+                                        'secretRef': {
+                                            'name': 'config-wxy'
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                }
+            },
+            'metadata': {
+                'namespace': 'apply',
+                'name': u'apply-deployment'
+            }
+        },
+        actual={
+            'kind': u'Deployment',
+            'spec': {
+                'replicas': 0,
+                'template': {
+                    'spec': {
+                        'containers': [
+                            {
+                                'name': 'this_must_exist',
+                                'envFrom': [
+                                    {
+                                        'configMapRef': {
+                                            'name': 'config-xyz'
+                                        }
+                                    },
+                                    {
+                                        'secretRef': {
+                                            'name': 'config-wxy'
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                }
+            },
+            'metadata': {
+                'namespace': 'apply',
+                'name': u'apply-deployment'
+            }
+        },
+        desired={
+            'kind': u'Deployment',
+            'spec': {
+                'replicas': 1,
+                'template': {
+                    'spec': {
+                        'containers': [
+                            {
+                                'name': 'this_must_exist',
+                                'envFrom': [
+                                    {
+                                        'configMapRef': {
+                                            'name': 'config-abc'
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                }
+            },
+            'metadata': {
+                'namespace': 'apply',
+                'name': u'apply-deployment'
+            }
+        },
+        expected={
+            'spec': {
+                'replicas': 1,
+                'template': {
+                    'spec': {
+                        'containers': [
+                            {
+                                'name': 'this_must_exist',
+                                'envFrom': [
+                                    {
+                                        'configMapRef': {
+                                            'name': 'config-abc'
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                }
+            }
+        }
+    ),
+    dict(
+        last_applied={
+            'kind': 'MadeUp',
+            'toplevel': {
+                'original': 'entry'
+            }
+        },
+        actual={
+            'kind': 'MadeUp',
+            'toplevel': {
+                'original': 'entry',
+                'another': {
+                    'nested': {
+                        'entry': 'value'
+                    }
+                }
+            }
+        },
+        desired={
+            'kind': 'MadeUp',
+            'toplevel': {
+                'original': 'entry',
+                'another': {
+                    'nested': {
+                        'entry': 'value'
+                    }
+                }
+            }
+        },
+        expected={}
+    )
+]
+
+
+def test_merges():
+    for test in tests:
+        assert(merge(test['last_applied'], test['desired'], test.get('actual', test['last_applied'])) == test['expected'])
+
+
+def test_apply_patch():
+    actual = dict(
+        kind="ConfigMap",
+        metadata=dict(name="foo",
+                      annotations={'kubectl.kubernetes.io/last-applied-configuration':
+                                   '{"data":{"one":"1","two":"2"},"kind":"ConfigMap",'
+                                   '"metadata":{"annotations":{"hello":"world","this":"one"},"name":"foo"}}',
+                                   'this': 'one', 'hello': 'world'}),
+        data=dict(one="1", two="2")
+    )
+    desired = dict(
+        kind="ConfigMap",
+        metadata=dict(name="foo"),
+        data=dict(one="1", three="3")
+    )
+    expected = dict(
+        metadata=dict(
+            annotations={'kubectl.kubernetes.io/last-applied-configuration': '{"data":{"one":"1","three":"3"},"kind":"ConfigMap","metadata":{"name":"foo"}}',
+                         'this': None, 'hello': None}),
+        data=dict(two=None, three="3")
+    )
+    assert(apply_patch(actual, desired) == (actual, expected))

tests/unit/requirements.txt

@@ -1,2 +1,3 @@
 pytest
 PyYAML
+kubernetes