merge with main branch

plugins/module_utils/cache.py

@@ -1,55 +0,0 @@
-# Copyright [2017] [Red Hat, Inc.]
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from __future__ import absolute_import, division, print_function
-__metaclass__ = type
-
-import os
-
-from ansible.module_utils.six import PY3
-
-
-def get_user():
-    if hasattr(os, 'getlogin'):
-        try:
-            user = os.getlogin()
-            if user:
-                return str(user)
-        except OSError:
-            pass
-    if hasattr(os, 'getuid'):
-        try:
-            user = os.getuid()
-            if user:
-                return str(user)
-        except OSError:
-            pass
-    user = os.environ.get("USERNAME")
-    if user:
-        return str(user)
-    return None
-
-
-def get_default_cache_id(client):
-    user = get_user()
-    if user:
-        cache_id = "{0}-{1}".format(client.configuration.host, user)
-    else:
-        cache_id = client.configuration.host
-
-    if PY3:
-        return cache_id.encode('utf-8')
-
-    return cache_id

plugins/module_utils/client/discovery.py

@@ -0,0 +1,154 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+from collections import defaultdict
+import hashlib
+import tempfile
+
+import kubernetes.dynamic
+import kubernetes.dynamic.discovery
+from kubernetes import __version__
+from kubernetes.dynamic.exceptions import ServiceUnavailableError
+
+from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import ResourceList
+
+
+class Discoverer(kubernetes.dynamic.discovery.Discoverer):
+    def __init__(self, client, cache_file):
+        self.client = client
+        default_cache_file_name = 'k8srcp-{0}.json'.format(hashlib.sha256(self.__get_default_cache_id()).hexdigest())
+        self.__cache_file = cache_file or os.path.join(tempfile.gettempdir(), default_cache_file_name)
+        self.__init_cache()
+
+    def __get_default_cache_id(self):
+        user = self.__get_user()
+        if user:
+            cache_id = "{0}-{1}".format(self.client.configuration.host, user)
+        else:
+            cache_id = self.client.configuration.host
+        return cache_id.encode('utf-8')
+
+    def __get_user(self):
+        # This is intended to provide a portable method for getting a username.
+        # It could, and maybe should, be replaced by getpass.getuser() but, due
+        # to a lack of portability testing the original code is being left in
+        # place.
+        if hasattr(os, 'getlogin'):
+            try:
+                user = os.getlogin()
+                if user:
+                    return str(user)
+            except OSError:
+                pass
+        if hasattr(os, 'getuid'):
+            try:
+                user = os.getuid()
+                if user:
+                    return str(user)
+            except OSError:
+                pass
+        user = os.environ.get("USERNAME")
+        if user:
+            return str(user)
+        return None
+
+    def __init_cache(self, refresh=False):
+        if refresh or not os.path.exists(self.__cache_file):
+            self._cache = {'library_version': __version__}
+            refresh = True
+        else:
+            try:
+                with open(self.__cache_file, 'r') as f:
+                    self._cache = json.load(f, cls=CacheDecoder(self.client))
+                if self._cache.get('library_version') != __version__:
+                    # Version mismatch, need to refresh cache
+                    self.invalidate_cache()
+            except Exception:
+                self.invalidate_cache()
+        self._load_server_info()
+        self.discover()
+        if refresh:
+            self._write_cache()
+
+    def get_resources_for_api_version(self, prefix, group, version, preferred):
+        """ returns a dictionary of resources associated with provided (prefix, group, version)"""
+
+        resources = defaultdict(list)
+        subresources = defaultdict(dict)
+
+        path = '/'.join(filter(None, [prefix, group, version]))
+        try:
+            resources_response = self.client.request('GET', path).resources or []
+        except ServiceUnavailableError:
+            resources_response = []
+
+        resources_raw = list(filter(lambda resource: '/' not in resource['name'], resources_response))
+        subresources_raw = list(filter(lambda resource: '/' in resource['name'], resources_response))
+        for subresource in subresources_raw:
+            resource, name = subresource['name'].split('/')
+            subresources[resource][name] = subresource
+
+        for resource in resources_raw:
+            # Prevent duplicate keys
+            for key in ('prefix', 'group', 'api_version', 'client', 'preferred'):
+                resource.pop(key, None)
+
+            resourceobj = kubernetes.dynamic.Resource(
+                prefix=prefix,
+                group=group,
+                api_version=version,
+                client=self.client,
+                preferred=preferred,
+                subresources=subresources.get(resource['name']),
+                **resource
+            )
+            resources[resource['kind']].append(resourceobj)
+
+            resource_lookup = {
+                'prefix': prefix,
+                'group': group,
+                'api_version': version,
+                'kind': resourceobj.kind,
+                'name': resourceobj.name
+            }
+            resource_list = ResourceList(self.client, group=group, api_version=version, base_kind=resource['kind'], base_resource_lookup=resource_lookup)
+            resources[resource_list.kind].append(resource_list)
+        return resources
+
+
+class LazyDiscoverer(Discoverer, kubernetes.dynamic.LazyDiscoverer):
+    def __init__(self, client, cache_file):
+        Discoverer.__init__(self, client, cache_file)
+        self.__update_cache = False
+
+
+class CacheDecoder(json.JSONDecoder):
+    def __init__(self, client, *args, **kwargs):
+        self.client = client
+        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, obj):
+        if '_type' not in obj:
+            return obj
+        _type = obj.pop('_type')
+        if _type == 'Resource':
+            return kubernetes.dynamic.Resource(client=self.client, **obj)
+        elif _type == 'ResourceList':
+            return ResourceList(self.client, **obj)
+        elif _type == 'ResourceGroup':
+            return kubernetes.dynamic.discovery.ResourceGroup(obj['preferred'], resources=self.object_hook(obj['resources']))
+        return obj

plugins/module_utils/client/resource.py

@@ -0,0 +1,49 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+import kubernetes.dynamic
+
+
+class ResourceList(kubernetes.dynamic.resource.ResourceList):
+    def __init__(self, client, group='', api_version='v1', base_kind='', kind=None, base_resource_lookup=None):
+        self.client = client
+        self.group = group
+        self.api_version = api_version
+        self.kind = kind or '{0}List'.format(base_kind)
+        self.base_kind = base_kind
+        self.base_resource_lookup = base_resource_lookup
+        self.__base_resource = None
+
+    def base_resource(self):
+        if self.__base_resource:
+            return self.__base_resource
+        elif self.base_resource_lookup:
+            self.__base_resource = self.client.resources.get(**self.base_resource_lookup)
+            return self.__base_resource
+        return None
+
+    def to_dict(self):
+        return {
+            '_type': 'ResourceList',
+            'group': self.group,
+            'api_version': self.api_version,
+            'kind': self.kind,
+            'base_kind': self.base_kind,
+            'base_resource_lookup': self.base_resource_lookup
+        }

plugins/module_utils/common.py

@@ -23,14 +23,12 @@ import time
 import os
 import traceback
 import sys
-import tempfile
 import hashlib
 from datetime import datetime
 from distutils.version import LooseVersion
 
 from ansible_collections.kubernetes.core.plugins.module_utils.args_common import (AUTH_ARG_MAP, AUTH_ARG_SPEC, AUTH_PROXY_HEADERS_SPEC)
 from ansible_collections.kubernetes.core.plugins.module_utils.hashes import generate_hash
-from ansible_collections.kubernetes.core.plugins.module_utils.cache import get_default_cache_id
 
 from ansible.module_utils.basic import missing_required_lib
 from ansible.module_utils.six import iteritems, string_types
@@ -42,10 +40,10 @@ from ansible.errors import AnsibleError
 K8S_IMP_ERR = None
 try:
     import kubernetes
-    import openshift
     from kubernetes.dynamic.exceptions import (
         NotFoundError, ResourceNotFoundError, ResourceNotUniqueError, DynamicApiError,
-        ConflictError, ForbiddenError, MethodNotAllowedError, BadRequestError
+        ConflictError, ForbiddenError, MethodNotAllowedError, BadRequestError,
+        KubernetesValidateMissing
     )
     HAS_K8S_MODULE_HELPER = True
     k8s_import_exception = None
@@ -57,6 +55,7 @@ except ImportError as e:
 IMP_K8S_CLIENT = None
 try:
     from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import K8SDynamicClient
+    from ansible_collections.kubernetes.core.plugins.module_utils.client.discovery import LazyDiscoverer
     IMP_K8S_CLIENT = True
 except ImportError as e:
     IMP_K8S_CLIENT = False
@@ -71,14 +70,6 @@ except ImportError:
     YAML_IMP_ERR = traceback.format_exc()
     HAS_YAML = False
 
-K8S_CONFIG_HASH_IMP_ERR = None
-try:
-    from kubernetes.dynamic.exceptions import KubernetesValidateMissing
-    HAS_K8S_CONFIG_HASH = True
-except ImportError:
-    K8S_CONFIG_HASH_IMP_ERR = traceback.format_exc()
-    HAS_K8S_CONFIG_HASH = False
-
 HAS_K8S_APPLY = None
 try:
     from ansible_collections.kubernetes.core.plugins.module_utils.apply import apply_object
@@ -114,6 +105,17 @@ except ImportError as e:
     HAS_JSONPATH_RW = False
     jsonpath_import_exception = e
 
+JSON_PATCH_IMP_ERR = None
+try:
+    import jsonpatch
+    HAS_JSON_PATCH = True
+    jsonpatch_import_exception = None
+except ImportError as e:
+    HAS_JSON_PATCH = False
+    jsonpatch_import_exception = e
+    JSON_PATCH_IMP_ERR = traceback.format_exc()
+
+
 def configuration_digest(configuration):
     m = hashlib.sha256()
     for k in AUTH_ARG_MAP:
@@ -211,15 +213,8 @@ def get_api_client(module=None, **kwargs):
         client = get_api_client._pool[digest]
         return client
 
-    def generate_cache_file(kubeclient):
-        cache_file_name = 'k8srcp-{0}.json'.format(hashlib.sha256(get_default_cache_id(kubeclient)).hexdigest())
-        return os.path.join(tempfile.gettempdir(), cache_file_name)
-
-    kubeclient = kubernetes.client.ApiClient(configuration)
-    cache_file = generate_cache_file(kubeclient)
-
     try:
-        client = K8SDynamicClient(kubeclient, cache_file)
+        client = K8SDynamicClient(kubernetes.client.ApiClient(configuration), discoverer=LazyDiscoverer)
     except Exception as err:
         _raise_or_fail(err, 'Failed to get client due to %s')
 
@@ -234,9 +229,9 @@ class K8sAnsibleMixin(object):
 
     def __init__(self, module, *args, **kwargs):
         if not HAS_K8S_MODULE_HELPER:
-            module.fail_json(msg=missing_required_lib('openshift'), exception=K8S_IMP_ERR,
+            module.fail_json(msg=missing_required_lib('kubernetes'), exception=K8S_IMP_ERR,
                              error=to_native(k8s_import_exception))
-        self.openshift_version = openshift.__version__
+        self.kubernetes_version = kubernetes.__version__
 
         if not HAS_YAML:
             module.fail_json(msg=missing_required_lib("PyYAML"), exception=YAML_IMP_ERR)
@@ -514,21 +509,8 @@ class K8sAnsibleMixin(object):
             self.resource_definitions = [implicit_definition]
 
     def check_library_version(self):
-        validate = self.params.get('validate')
-        if validate and LooseVersion(self.openshift_version) < LooseVersion("0.8.0"):
-            self.fail_json(msg="openshift >= 0.8.0 is required for validate")
-        self.append_hash = self.params.get('append_hash')
-        if self.append_hash and not HAS_K8S_CONFIG_HASH:
-            self.fail_json(msg=missing_required_lib("openshift >= 0.7.2", reason="for append_hash"),
-                           exception=K8S_CONFIG_HASH_IMP_ERR)
-        if self.params['merge_type'] and LooseVersion(self.openshift_version) < LooseVersion("0.6.2"):
-            self.fail_json(msg=missing_required_lib("openshift >= 0.6.2", reason="for merge_type"))
-        self.apply = self.params.get('apply', False)
-        if self.apply and not HAS_K8S_APPLY:
-            self.fail_json(msg=missing_required_lib("openshift >= 0.9.2", reason="for apply"))
-        wait = self.params.get('wait', False)
-        if wait and not HAS_K8S_INSTANCE_HELPER:
-            self.fail_json(msg=missing_required_lib("openshift >= 0.4.0", reason="for wait"))
+        if LooseVersion(self.kubernetes_version) < LooseVersion("12.0.0"):
+            self.fail_json(msg="kubernetes >= 12.0.0 is required")
 
     def flatten_list_kind(self, list_resource, definitions):
         flattened = []
@@ -609,6 +591,8 @@ class K8sAnsibleMixin(object):
         return definition
 
     def perform_action(self, resource, definition):
+        append_hash = self.params.get('append_hash', False)
+        apply = self.params.get('apply', False)
         delete_options = self.params.get('delete_options')
         result = {'changed': False, 'result': {}}
         state = self.params.get('state', None)
@@ -633,7 +617,7 @@ class K8sAnsibleMixin(object):
 
         try:
             # ignore append_hash for resources other than ConfigMap and Secret
-            if self.append_hash and definition['kind'] in ['ConfigMap', 'Secret']:
+            if append_hash and definition['kind'] in ['ConfigMap', 'Secret']:
                 name = '%s-%s' % (name, generate_hash(definition))
                 definition['metadata']['name'] = name
             params = dict(name=name)
@@ -710,7 +694,7 @@ class K8sAnsibleMixin(object):
                                 self.fail_json(msg=build_error_msg(definition['kind'], origin_name, msg), **result)
                 return result
         else:
-            if self.apply:
+            if apply:
                 if self.check_mode:
                     ignored, patch = apply_object(resource, _encode_stringdata(definition))
                     if existing:
@@ -806,7 +790,7 @@ class K8sAnsibleMixin(object):
                     k8s_obj = _encode_stringdata(definition)
                 else:
                     try:
-                        k8s_obj = resource.replace(definition, name=name, namespace=namespace, append_hash=self.append_hash).to_dict()
+                        k8s_obj = resource.replace(definition, name=name, namespace=namespace, append_hash=append_hash).to_dict()
                     except DynamicApiError as exc:
                         msg = "Failed to replace object: {0}".format(exc.body)
                         if self.warnings:
@@ -839,15 +823,11 @@ class K8sAnsibleMixin(object):
             if self.check_mode:
                 k8s_obj = dict_merge(existing.to_dict(), _encode_stringdata(definition))
             else:
-                if LooseVersion(self.openshift_version) < LooseVersion("0.6.2"):
+                for merge_type in self.params['merge_type'] or ['strategic-merge', 'merge']:
                     k8s_obj, error = self.patch_resource(resource, definition, existing, name,
-                                                         namespace)
-                else:
-                    for merge_type in self.params['merge_type'] or ['strategic-merge', 'merge']:
-                        k8s_obj, error = self.patch_resource(resource, definition, existing, name,
-                                                             namespace, merge_type=merge_type)
-                        if not error:
-                            break
+                                                         namespace, merge_type=merge_type)
+                    if not error:
+                        break
                 if error:
                     if continue_on_error:
                         result['error'] = error
@@ -874,12 +854,42 @@ class K8sAnsibleMixin(object):
                     self.fail_json(msg=msg, **result)
             return result
 
+    def json_patch(self, existing, definition, merge_type):
+        if merge_type == "json":
+            if not HAS_JSON_PATCH:
+                error = {
+                    "msg": missing_required_lib('jsonpatch'),
+                    "exception": JSON_PATCH_IMP_ERR,
+                    "error": to_native(jsonpatch_import_exception)
+                }
+                return None, error
+            try:
+                patch = jsonpatch.JsonPatch([definition])
+                result_patch = patch.apply(existing.to_dict())
+                return result_patch, None
+            except jsonpatch.InvalidJsonPatch as e:
+                error = {
+                    "msg": "invalid json patch",
+                    "error": to_native(e)
+                }
+                return None, error
+            except jsonpatch.JsonPatchConflict as e:
+                error = {
+                    "msg": "patch could not be applied due to conflict situation",
+                    "error": to_native(e)
+                }
+                return None, error
+        return definition, None
+
     def patch_resource(self, resource, definition, existing, name, namespace, merge_type=None):
         try:
             params = dict(name=name, namespace=namespace)
             if merge_type:
                 params['content_type'] = 'application/{0}-patch+json'.format(merge_type)
-            k8s_obj = resource.patch(definition, **params).to_dict()
+            patch_data, error = self.json_patch(existing, definition, merge_type)
+            if error is not None:
+                return None, error
+            k8s_obj = resource.patch(patch_data, **params).to_dict()
             match, diffs = self.diff_objects(existing.to_dict(), k8s_obj)
             error = {}
             return k8s_obj, {}