Replicate base resource for lists functionality (#89)

* Replicate base resource for lists functionality

This replicates specific functionality from the openshift client to more
reliably retrieve the base resource from a resource list.

* Add changelog fragment

changelogs/fragments/89-replicate-base-resource.yaml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+  - replicate base resource for lists functionality (https://github.com/ansible-collections/kubernetes.core/pull/89).

plugins/module_utils/cache.py

@@ -1,55 +0,0 @@
-# Copyright [2017] [Red Hat, Inc.]
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from __future__ import absolute_import, division, print_function
-__metaclass__ = type
-
-import os
-
-from ansible.module_utils.six import PY3
-
-
-def get_user():
-    if hasattr(os, 'getlogin'):
-        try:
-            user = os.getlogin()
-            if user:
-                return str(user)
-        except OSError:
-            pass
-    if hasattr(os, 'getuid'):
-        try:
-            user = os.getuid()
-            if user:
-                return str(user)
-        except OSError:
-            pass
-    user = os.environ.get("USERNAME")
-    if user:
-        return str(user)
-    return None
-
-
-def get_default_cache_id(client):
-    user = get_user()
-    if user:
-        cache_id = "{0}-{1}".format(client.configuration.host, user)
-    else:
-        cache_id = client.configuration.host
-
-    if PY3:
-        return cache_id.encode('utf-8')
-
-    return cache_id

plugins/module_utils/client/discovery.py

@@ -0,0 +1,154 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+from collections import defaultdict
+import hashlib
+import tempfile
+
+import kubernetes.dynamic
+import kubernetes.dynamic.discovery
+from kubernetes import __version__
+from kubernetes.dynamic.exceptions import ServiceUnavailableError
+
+from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import ResourceList
+
+
+class Discoverer(kubernetes.dynamic.discovery.Discoverer):
+    def __init__(self, client, cache_file):
+        self.client = client
+        default_cache_file_name = 'k8srcp-{0}.json'.format(hashlib.sha256(self.__get_default_cache_id()).hexdigest())
+        self.__cache_file = cache_file or os.path.join(tempfile.gettempdir(), default_cache_file_name)
+        self.__init_cache()
+
+    def __get_default_cache_id(self):
+        user = self.__get_user()
+        if user:
+            cache_id = "{0}-{1}".format(self.client.configuration.host, user)
+        else:
+            cache_id = self.client.configuration.host
+        return cache_id.encode('utf-8')
+
+    def __get_user(self):
+        # This is intended to provide a portable method for getting a username.
+        # It could, and maybe should, be replaced by getpass.getuser() but, due
+        # to a lack of portability testing the original code is being left in
+        # place.
+        if hasattr(os, 'getlogin'):
+            try:
+                user = os.getlogin()
+                if user:
+                    return str(user)
+            except OSError:
+                pass
+        if hasattr(os, 'getuid'):
+            try:
+                user = os.getuid()
+                if user:
+                    return str(user)
+            except OSError:
+                pass
+        user = os.environ.get("USERNAME")
+        if user:
+            return str(user)
+        return None
+
+    def __init_cache(self, refresh=False):
+        if refresh or not os.path.exists(self.__cache_file):
+            self._cache = {'library_version': __version__}
+            refresh = True
+        else:
+            try:
+                with open(self.__cache_file, 'r') as f:
+                    self._cache = json.load(f, cls=CacheDecoder(self.client))
+                if self._cache.get('library_version') != __version__:
+                    # Version mismatch, need to refresh cache
+                    self.invalidate_cache()
+            except Exception:
+                self.invalidate_cache()
+        self._load_server_info()
+        self.discover()
+        if refresh:
+            self._write_cache()
+
+    def get_resources_for_api_version(self, prefix, group, version, preferred):
+        """ returns a dictionary of resources associated with provided (prefix, group, version)"""
+
+        resources = defaultdict(list)
+        subresources = defaultdict(dict)
+
+        path = '/'.join(filter(None, [prefix, group, version]))
+        try:
+            resources_response = self.client.request('GET', path).resources or []
+        except ServiceUnavailableError:
+            resources_response = []
+
+        resources_raw = list(filter(lambda resource: '/' not in resource['name'], resources_response))
+        subresources_raw = list(filter(lambda resource: '/' in resource['name'], resources_response))
+        for subresource in subresources_raw:
+            resource, name = subresource['name'].split('/')
+            subresources[resource][name] = subresource
+
+        for resource in resources_raw:
+            # Prevent duplicate keys
+            for key in ('prefix', 'group', 'api_version', 'client', 'preferred'):
+                resource.pop(key, None)
+
+            resourceobj = kubernetes.dynamic.Resource(
+                prefix=prefix,
+                group=group,
+                api_version=version,
+                client=self.client,
+                preferred=preferred,
+                subresources=subresources.get(resource['name']),
+                **resource
+            )
+            resources[resource['kind']].append(resourceobj)
+
+            resource_lookup = {
+                'prefix': prefix,
+                'group': group,
+                'api_version': version,
+                'kind': resourceobj.kind,
+                'name': resourceobj.name
+            }
+            resource_list = ResourceList(self.client, group=group, api_version=version, base_kind=resource['kind'], base_resource_lookup=resource_lookup)
+            resources[resource_list.kind].append(resource_list)
+        return resources
+
+
+class LazyDiscoverer(Discoverer, kubernetes.dynamic.LazyDiscoverer):
+    def __init__(self, client, cache_file):
+        Discoverer.__init__(self, client, cache_file)
+        self.__update_cache = False
+
+
+class CacheDecoder(json.JSONDecoder):
+    def __init__(self, client, *args, **kwargs):
+        self.client = client
+        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, obj):
+        if '_type' not in obj:
+            return obj
+        _type = obj.pop('_type')
+        if _type == 'Resource':
+            return kubernetes.dynamic.Resource(client=self.client, **obj)
+        elif _type == 'ResourceList':
+            return ResourceList(self.client, **obj)
+        elif _type == 'ResourceGroup':
+            return kubernetes.dynamic.discovery.ResourceGroup(obj['preferred'], resources=self.object_hook(obj['resources']))
+        return obj

plugins/module_utils/client/resource.py

@@ -0,0 +1,49 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+import kubernetes.dynamic
+
+
+class ResourceList(kubernetes.dynamic.resource.ResourceList):
+    def __init__(self, client, group='', api_version='v1', base_kind='', kind=None, base_resource_lookup=None):
+        self.client = client
+        self.group = group
+        self.api_version = api_version
+        self.kind = kind or '{0}List'.format(base_kind)
+        self.base_kind = base_kind
+        self.base_resource_lookup = base_resource_lookup
+        self.__base_resource = None
+
+    def base_resource(self):
+        if self.__base_resource:
+            return self.__base_resource
+        elif self.base_resource_lookup:
+            self.__base_resource = self.client.resources.get(**self.base_resource_lookup)
+            return self.__base_resource
+        return None
+
+    def to_dict(self):
+        return {
+            '_type': 'ResourceList',
+            'group': self.group,
+            'api_version': self.api_version,
+            'kind': self.kind,
+            'base_kind': self.base_kind,
+            'base_resource_lookup': self.base_resource_lookup
+        }

plugins/module_utils/common.py

@@ -23,14 +23,12 @@ import time
 import os
 import traceback
 import sys
-import tempfile
 import hashlib
 from datetime import datetime
 from distutils.version import LooseVersion
 
 from ansible_collections.kubernetes.core.plugins.module_utils.args_common import (AUTH_ARG_MAP, AUTH_ARG_SPEC, AUTH_PROXY_HEADERS_SPEC)
 from ansible_collections.kubernetes.core.plugins.module_utils.hashes import generate_hash
-from ansible_collections.kubernetes.core.plugins.module_utils.cache import get_default_cache_id
 
 from ansible.module_utils.basic import missing_required_lib
 from ansible.module_utils.six import iteritems, string_types
@@ -56,6 +54,7 @@ except ImportError as e:
 IMP_K8S_CLIENT = None
 try:
     from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import K8SDynamicClient
+    from ansible_collections.kubernetes.core.plugins.module_utils.client.discovery import LazyDiscoverer
     IMP_K8S_CLIENT = True
 except ImportError as e:
     IMP_K8S_CLIENT = False
@@ -214,15 +213,8 @@ def get_api_client(module=None, **kwargs):
         client = get_api_client._pool[digest]
         return client
 
-    def generate_cache_file(kubeclient):
-        cache_file_name = 'k8srcp-{0}.json'.format(hashlib.sha256(get_default_cache_id(kubeclient)).hexdigest())
-        return os.path.join(tempfile.gettempdir(), cache_file_name)
-
-    kubeclient = kubernetes.client.ApiClient(configuration)
-    cache_file = generate_cache_file(kubeclient)
-
     try:
-        client = K8SDynamicClient(kubeclient, cache_file)
+        client = K8SDynamicClient(kubernetes.client.ApiClient(configuration), discoverer=LazyDiscoverer)
     except Exception as err:
         _raise_or_fail(err, 'Failed to get client due to %s')
 

plugins/modules/k8s_cluster_info.py

@@ -140,19 +140,29 @@ apis:
 
 
 import copy
-
-from ansible_collections.kubernetes.core.plugins.module_utils.ansiblemodule import AnsibleModule
-from ansible.module_utils.parsing.convert_bool import boolean
-from ansible_collections.kubernetes.core.plugins.module_utils.args_common import (AUTH_ARG_SPEC)
+import traceback
 from collections import defaultdict
 
+HAS_K8S = False
+try:
+    from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import ResourceList
+    HAS_K8S = True
+except ImportError as e:
+    K8S_IMP_ERR = e
+    K8S_IMP_EXC = traceback.format_exc()
+
+from ansible.module_utils._text import to_native
+from ansible.module_utils.basic import missing_required_lib
+from ansible.module_utils.parsing.convert_bool import boolean
+from ansible_collections.kubernetes.core.plugins.module_utils.ansiblemodule import AnsibleModule
+from ansible_collections.kubernetes.core.plugins.module_utils.args_common import (AUTH_ARG_SPEC)
+
 
 def execute_module(module, client):
     invalidate_cache = boolean(module.params.get('invalidate_cache', True), strict=False)
     if invalidate_cache:
         client.resources.invalidate_cache()
     results = defaultdict(dict)
-    from openshift.dynamic.resource import ResourceList
     for resource in list(client.resources):
         resource = resource[0]
         if isinstance(resource, ResourceList):
@@ -192,6 +202,9 @@ def argspec():
 
 def main():
     module = AnsibleModule(argument_spec=argspec(), supports_check_mode=True)
+    if not HAS_K8S:
+        module.fail_json(msg=missing_required_lib('kubernetes'), exception=K8S_IMP_EXC,
+                         error=to_native(K8S_IMP_ERR))
     from ansible_collections.kubernetes.core.plugins.module_utils.common import get_api_client
     execute_module(module, client=get_api_client(module=module))
 

tests/sanity/ignore-2.10.txt

@@ -9,4 +9,12 @@ molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml
 molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 plugins/module_utils/k8sdynamicclient.py import-2.7!skip
-plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/client/discovery.py import-2.7!skip
+plugins/module_utils/client/discovery.py import-3.7!skip
+plugins/module_utils/client/resource.py import-2.7!skip
+plugins/module_utils/client/resource.py import-3.7!skip
+plugins/module_utils/client/discovery.py future-import-boilerplate!skip
+plugins/module_utils/client/discovery.py metaclass-boilerplate!skip
+tests/unit/module_utils/test_discoverer.py future-import-boilerplate!skip
+tests/unit/module_utils/test_discoverer.py metaclass-boilerplate!skip

tests/sanity/ignore-2.11.txt

@@ -9,4 +9,12 @@ molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml
 molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 plugins/module_utils/k8sdynamicclient.py import-2.7!skip
-plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/client/discovery.py import-2.7!skip
+plugins/module_utils/client/discovery.py import-3.7!skip
+plugins/module_utils/client/resource.py import-2.7!skip
+plugins/module_utils/client/resource.py import-3.7!skip
+plugins/module_utils/client/discovery.py future-import-boilerplate!skip
+plugins/module_utils/client/discovery.py metaclass-boilerplate!skip
+tests/unit/module_utils/test_discoverer.py future-import-boilerplate!skip
+tests/unit/module_utils/test_discoverer.py metaclass-boilerplate!skip

tests/sanity/ignore-2.12.txt

@@ -10,3 +10,9 @@ molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllin
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 plugins/module_utils/k8sdynamicclient.py import-2.7!skip
 plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/client/discovery.py import-2.7!skip
+plugins/module_utils/client/discovery.py import-3.7!skip
+plugins/module_utils/client/resource.py import-2.7!skip
+plugins/module_utils/client/resource.py import-3.7!skip
+plugins/module_utils/client/discovery.py future-import-boilerplate!skip
+plugins/module_utils/client/discovery.py metaclass-boilerplate!skip

tests/sanity/ignore-2.9.txt

@@ -6,4 +6,8 @@ molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml
 molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
 molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 plugins/module_utils/k8sdynamicclient.py import-2.7!skip
-plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/client/discovery.py future-import-boilerplate!skip
+plugins/module_utils/client/discovery.py metaclass-boilerplate!skip
+tests/unit/module_utils/test_discoverer.py future-import-boilerplate!skip
+tests/unit/module_utils/test_discoverer.py metaclass-boilerplate!skip

tests/unit/module_utils/test_discoverer.py

@@ -0,0 +1,143 @@
+# Copyright [2017] [Red Hat, Inc.]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import pytest
+
+from kubernetes.client import ApiClient
+from kubernetes.dynamic import Resource
+
+from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import K8SDynamicClient
+from ansible_collections.kubernetes.core.plugins.module_utils.client.discovery import LazyDiscoverer
+from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import ResourceList
+
+
+@pytest.fixture(scope='module')
+def mock_namespace():
+    return Resource(
+        api_version='v1',
+        kind='Namespace',
+        name='namespaces',
+        namespaced=False,
+        preferred=True,
+        prefix='api',
+        shorter_names=['ns'],
+        shortNames=['ns'],
+        singularName='namespace',
+        verbs=['create', 'delete', 'get', 'list', 'patch', 'update', 'watch']
+    )
+
+
+@pytest.fixture(scope='module')
+def mock_templates():
+    return Resource(
+        api_version='v1',
+        kind='Template',
+        name='templates',
+        namespaced=True,
+        preferred=True,
+        prefix='api',
+        shorter_names=[],
+        shortNames=[],
+        verbs=['create', 'delete', 'get', 'list', 'patch', 'update', 'watch']
+    )
+
+
+@pytest.fixture(scope='module')
+def mock_processedtemplates():
+    return Resource(
+        api_version='v1',
+        kind='Template',
+        name='processedtemplates',
+        namespaced=True,
+        preferred=True,
+        prefix='api',
+        shorter_names=[],
+        shortNames=[],
+        verbs=['create', 'delete', 'get', 'list', 'patch', 'update', 'watch']
+    )
+
+
+@pytest.fixture(scope='module')
+def mock_namespace_list(mock_namespace):
+    ret = ResourceList(mock_namespace.client, mock_namespace.group, mock_namespace.api_version, mock_namespace.kind)
+    ret._ResourceList__base_resource = mock_namespace
+    return ret
+
+
+@pytest.fixture(scope='function', autouse=True)
+def setup_client_monkeypatch(monkeypatch, mock_namespace, mock_namespace_list, mock_templates, mock_processedtemplates):
+
+    def mock_load_server_info(self):
+        self.__version = {'kubernetes': 'mock-k8s-version'}
+
+    def mock_parse_api_groups(self, request_resources=False):
+        return {
+            'api': {
+                '': {
+                    'v1': {
+                        'Namespace': [mock_namespace],
+                        'NamespaceList': [mock_namespace_list],
+                        'Template': [mock_templates, mock_processedtemplates],
+                    }
+                }
+            }
+        }
+
+    monkeypatch.setattr(LazyDiscoverer, '_load_server_info', mock_load_server_info)
+    monkeypatch.setattr(LazyDiscoverer, 'parse_api_groups', mock_parse_api_groups)
+
+
+@pytest.fixture
+def client(request):
+    return K8SDynamicClient(ApiClient(), discoverer=LazyDiscoverer)
+
+
+@pytest.mark.parametrize(("attribute", "value"), [
+    ('name', 'namespaces'),
+    ('singular_name', 'namespace'),
+    ('short_names', ['ns'])
+])
+def test_search_returns_single_and_list(client, mock_namespace, mock_namespace_list, attribute, value):
+    resources = client.resources.search(**{'api_version': 'v1', attribute: value})
+
+    assert len(resources) == 2
+    assert mock_namespace in resources
+    assert mock_namespace_list in resources
+
+
+@pytest.mark.parametrize(("attribute", "value"), [
+    ('kind', 'Namespace'),
+    ('name', 'namespaces'),
+    ('singular_name', 'namespace'),
+    ('short_names', ['ns'])
+])
+def test_get_returns_only_single(client, mock_namespace, attribute, value):
+    resource = client.resources.get(**{'api_version': 'v1', attribute: value})
+
+    assert resource == mock_namespace
+
+
+def test_get_namespace_list_kind(client, mock_namespace_list):
+    resource = client.resources.get(api_version='v1', kind='NamespaceList')
+
+    assert resource == mock_namespace_list
+
+
+def test_search_multiple_resources_for_template(client, mock_templates, mock_processedtemplates):
+    resources = client.resources.search(api_version='v1', kind='Template')
+
+    assert len(resources) == 2
+    assert mock_templates in resources
+    assert mock_processedtemplates in resources