diff --git a/changelogs/fragments/89-replicate-base-resource.yaml b/changelogs/fragments/89-replicate-base-resource.yaml new file mode 100644 index 00000000..cf81e081 --- /dev/null +++ b/changelogs/fragments/89-replicate-base-resource.yaml @@ -0,0 +1,3 @@ +--- +minor_changes: + - replicate base resource for lists functionality (https://github.com/ansible-collections/kubernetes.core/pull/89). diff --git a/plugins/module_utils/cache.py b/plugins/module_utils/cache.py deleted file mode 100644 index 963c3d60..00000000 --- a/plugins/module_utils/cache.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright [2017] [Red Hat, Inc.] -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import absolute_import, division, print_function -__metaclass__ = type - -import os - -from ansible.module_utils.six import PY3 - - -def get_user(): - if hasattr(os, 'getlogin'): - try: - user = os.getlogin() - if user: - return str(user) - except OSError: - pass - if hasattr(os, 'getuid'): - try: - user = os.getuid() - if user: - return str(user) - except OSError: - pass - user = os.environ.get("USERNAME") - if user: - return str(user) - return None - - -def get_default_cache_id(client): - user = get_user() - if user: - cache_id = "{0}-{1}".format(client.configuration.host, user) - else: - cache_id = client.configuration.host - - if PY3: - return cache_id.encode('utf-8') - - return cache_id diff --git a/plugins/module_utils/client/discovery.py b/plugins/module_utils/client/discovery.py new file mode 100644 index 00000000..ec01f195 --- /dev/null +++ b/plugins/module_utils/client/discovery.py @@ -0,0 +1,154 @@ +# Copyright [2017] [Red Hat, Inc.] +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import os +from collections import defaultdict +import hashlib +import tempfile + +import kubernetes.dynamic +import kubernetes.dynamic.discovery +from kubernetes import __version__ +from kubernetes.dynamic.exceptions import ServiceUnavailableError + +from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import ResourceList + + +class Discoverer(kubernetes.dynamic.discovery.Discoverer): + def __init__(self, client, cache_file): + self.client = client + default_cache_file_name = 'k8srcp-{0}.json'.format(hashlib.sha256(self.__get_default_cache_id()).hexdigest()) + self.__cache_file = cache_file or os.path.join(tempfile.gettempdir(), default_cache_file_name) + self.__init_cache() + + def __get_default_cache_id(self): + user = self.__get_user() + if user: + cache_id = "{0}-{1}".format(self.client.configuration.host, user) + else: + cache_id = self.client.configuration.host + return cache_id.encode('utf-8') + + def __get_user(self): + # This is intended to provide a portable method for getting a username. + # It could, and maybe should, be replaced by getpass.getuser() but, due + # to a lack of portability testing the original code is being left in + # place. + if hasattr(os, 'getlogin'): + try: + user = os.getlogin() + if user: + return str(user) + except OSError: + pass + if hasattr(os, 'getuid'): + try: + user = os.getuid() + if user: + return str(user) + except OSError: + pass + user = os.environ.get("USERNAME") + if user: + return str(user) + return None + + def __init_cache(self, refresh=False): + if refresh or not os.path.exists(self.__cache_file): + self._cache = {'library_version': __version__} + refresh = True + else: + try: + with open(self.__cache_file, 'r') as f: + self._cache = json.load(f, cls=CacheDecoder(self.client)) + if self._cache.get('library_version') != __version__: + # Version mismatch, need to refresh cache + self.invalidate_cache() + except Exception: + self.invalidate_cache() + self._load_server_info() + self.discover() + if refresh: + self._write_cache() + + def get_resources_for_api_version(self, prefix, group, version, preferred): + """ returns a dictionary of resources associated with provided (prefix, group, version)""" + + resources = defaultdict(list) + subresources = defaultdict(dict) + + path = '/'.join(filter(None, [prefix, group, version])) + try: + resources_response = self.client.request('GET', path).resources or [] + except ServiceUnavailableError: + resources_response = [] + + resources_raw = list(filter(lambda resource: '/' not in resource['name'], resources_response)) + subresources_raw = list(filter(lambda resource: '/' in resource['name'], resources_response)) + for subresource in subresources_raw: + resource, name = subresource['name'].split('/') + subresources[resource][name] = subresource + + for resource in resources_raw: + # Prevent duplicate keys + for key in ('prefix', 'group', 'api_version', 'client', 'preferred'): + resource.pop(key, None) + + resourceobj = kubernetes.dynamic.Resource( + prefix=prefix, + group=group, + api_version=version, + client=self.client, + preferred=preferred, + subresources=subresources.get(resource['name']), + **resource + ) + resources[resource['kind']].append(resourceobj) + + resource_lookup = { + 'prefix': prefix, + 'group': group, + 'api_version': version, + 'kind': resourceobj.kind, + 'name': resourceobj.name + } + resource_list = ResourceList(self.client, group=group, api_version=version, base_kind=resource['kind'], base_resource_lookup=resource_lookup) + resources[resource_list.kind].append(resource_list) + return resources + + +class LazyDiscoverer(Discoverer, kubernetes.dynamic.LazyDiscoverer): + def __init__(self, client, cache_file): + Discoverer.__init__(self, client, cache_file) + self.__update_cache = False + + +class CacheDecoder(json.JSONDecoder): + def __init__(self, client, *args, **kwargs): + self.client = client + json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + + def object_hook(self, obj): + if '_type' not in obj: + return obj + _type = obj.pop('_type') + if _type == 'Resource': + return kubernetes.dynamic.Resource(client=self.client, **obj) + elif _type == 'ResourceList': + return ResourceList(self.client, **obj) + elif _type == 'ResourceGroup': + return kubernetes.dynamic.discovery.ResourceGroup(obj['preferred'], resources=self.object_hook(obj['resources'])) + return obj diff --git a/plugins/module_utils/client/resource.py b/plugins/module_utils/client/resource.py new file mode 100644 index 00000000..39d8a1cf --- /dev/null +++ b/plugins/module_utils/client/resource.py @@ -0,0 +1,49 @@ +# Copyright [2017] [Red Hat, Inc.] +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +import kubernetes.dynamic + + +class ResourceList(kubernetes.dynamic.resource.ResourceList): + def __init__(self, client, group='', api_version='v1', base_kind='', kind=None, base_resource_lookup=None): + self.client = client + self.group = group + self.api_version = api_version + self.kind = kind or '{0}List'.format(base_kind) + self.base_kind = base_kind + self.base_resource_lookup = base_resource_lookup + self.__base_resource = None + + def base_resource(self): + if self.__base_resource: + return self.__base_resource + elif self.base_resource_lookup: + self.__base_resource = self.client.resources.get(**self.base_resource_lookup) + return self.__base_resource + return None + + def to_dict(self): + return { + '_type': 'ResourceList', + 'group': self.group, + 'api_version': self.api_version, + 'kind': self.kind, + 'base_kind': self.base_kind, + 'base_resource_lookup': self.base_resource_lookup + } diff --git a/plugins/module_utils/common.py b/plugins/module_utils/common.py index 203fa4a9..9a7cb269 100644 --- a/plugins/module_utils/common.py +++ b/plugins/module_utils/common.py @@ -23,14 +23,12 @@ import time import os import traceback import sys -import tempfile import hashlib from datetime import datetime from distutils.version import LooseVersion from ansible_collections.kubernetes.core.plugins.module_utils.args_common import (AUTH_ARG_MAP, AUTH_ARG_SPEC, AUTH_PROXY_HEADERS_SPEC) from ansible_collections.kubernetes.core.plugins.module_utils.hashes import generate_hash -from ansible_collections.kubernetes.core.plugins.module_utils.cache import get_default_cache_id from ansible.module_utils.basic import missing_required_lib from ansible.module_utils.six import iteritems, string_types @@ -56,6 +54,7 @@ except ImportError as e: IMP_K8S_CLIENT = None try: from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import K8SDynamicClient + from ansible_collections.kubernetes.core.plugins.module_utils.client.discovery import LazyDiscoverer IMP_K8S_CLIENT = True except ImportError as e: IMP_K8S_CLIENT = False @@ -214,15 +213,8 @@ def get_api_client(module=None, **kwargs): client = get_api_client._pool[digest] return client - def generate_cache_file(kubeclient): - cache_file_name = 'k8srcp-{0}.json'.format(hashlib.sha256(get_default_cache_id(kubeclient)).hexdigest()) - return os.path.join(tempfile.gettempdir(), cache_file_name) - - kubeclient = kubernetes.client.ApiClient(configuration) - cache_file = generate_cache_file(kubeclient) - try: - client = K8SDynamicClient(kubeclient, cache_file) + client = K8SDynamicClient(kubernetes.client.ApiClient(configuration), discoverer=LazyDiscoverer) except Exception as err: _raise_or_fail(err, 'Failed to get client due to %s') diff --git a/plugins/modules/k8s_cluster_info.py b/plugins/modules/k8s_cluster_info.py index 3742777c..04cc09d8 100644 --- a/plugins/modules/k8s_cluster_info.py +++ b/plugins/modules/k8s_cluster_info.py @@ -140,19 +140,29 @@ apis: import copy - -from ansible_collections.kubernetes.core.plugins.module_utils.ansiblemodule import AnsibleModule -from ansible.module_utils.parsing.convert_bool import boolean -from ansible_collections.kubernetes.core.plugins.module_utils.args_common import (AUTH_ARG_SPEC) +import traceback from collections import defaultdict +HAS_K8S = False +try: + from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import ResourceList + HAS_K8S = True +except ImportError as e: + K8S_IMP_ERR = e + K8S_IMP_EXC = traceback.format_exc() + +from ansible.module_utils._text import to_native +from ansible.module_utils.basic import missing_required_lib +from ansible.module_utils.parsing.convert_bool import boolean +from ansible_collections.kubernetes.core.plugins.module_utils.ansiblemodule import AnsibleModule +from ansible_collections.kubernetes.core.plugins.module_utils.args_common import (AUTH_ARG_SPEC) + def execute_module(module, client): invalidate_cache = boolean(module.params.get('invalidate_cache', True), strict=False) if invalidate_cache: client.resources.invalidate_cache() results = defaultdict(dict) - from openshift.dynamic.resource import ResourceList for resource in list(client.resources): resource = resource[0] if isinstance(resource, ResourceList): @@ -192,6 +202,9 @@ def argspec(): def main(): module = AnsibleModule(argument_spec=argspec(), supports_check_mode=True) + if not HAS_K8S: + module.fail_json(msg=missing_required_lib('kubernetes'), exception=K8S_IMP_EXC, + error=to_native(K8S_IMP_ERR)) from ansible_collections.kubernetes.core.plugins.module_utils.common import get_api_client execute_module(module, client=get_api_client(module=module)) diff --git a/tests/sanity/ignore-2.10.txt b/tests/sanity/ignore-2.10.txt index c1468787..728285d1 100644 --- a/tests/sanity/ignore-2.10.txt +++ b/tests/sanity/ignore-2.10.txt @@ -9,4 +9,12 @@ molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip plugins/module_utils/k8sdynamicclient.py import-2.7!skip -plugins/module_utils/k8sdynamicclient.py import-3.7!skip \ No newline at end of file +plugins/module_utils/k8sdynamicclient.py import-3.7!skip +plugins/module_utils/client/discovery.py import-2.7!skip +plugins/module_utils/client/discovery.py import-3.7!skip +plugins/module_utils/client/resource.py import-2.7!skip +plugins/module_utils/client/resource.py import-3.7!skip +plugins/module_utils/client/discovery.py future-import-boilerplate!skip +plugins/module_utils/client/discovery.py metaclass-boilerplate!skip +tests/unit/module_utils/test_discoverer.py future-import-boilerplate!skip +tests/unit/module_utils/test_discoverer.py metaclass-boilerplate!skip diff --git a/tests/sanity/ignore-2.11.txt b/tests/sanity/ignore-2.11.txt index c1468787..728285d1 100644 --- a/tests/sanity/ignore-2.11.txt +++ b/tests/sanity/ignore-2.11.txt @@ -9,4 +9,12 @@ molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip plugins/module_utils/k8sdynamicclient.py import-2.7!skip -plugins/module_utils/k8sdynamicclient.py import-3.7!skip \ No newline at end of file +plugins/module_utils/k8sdynamicclient.py import-3.7!skip +plugins/module_utils/client/discovery.py import-2.7!skip +plugins/module_utils/client/discovery.py import-3.7!skip +plugins/module_utils/client/resource.py import-2.7!skip +plugins/module_utils/client/resource.py import-3.7!skip +plugins/module_utils/client/discovery.py future-import-boilerplate!skip +plugins/module_utils/client/discovery.py metaclass-boilerplate!skip +tests/unit/module_utils/test_discoverer.py future-import-boilerplate!skip +tests/unit/module_utils/test_discoverer.py metaclass-boilerplate!skip diff --git a/tests/sanity/ignore-2.12.txt b/tests/sanity/ignore-2.12.txt index dd0b476b..8b748a40 100644 --- a/tests/sanity/ignore-2.12.txt +++ b/tests/sanity/ignore-2.12.txt @@ -10,3 +10,9 @@ molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllin molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip plugins/module_utils/k8sdynamicclient.py import-2.7!skip plugins/module_utils/k8sdynamicclient.py import-3.7!skip +plugins/module_utils/client/discovery.py import-2.7!skip +plugins/module_utils/client/discovery.py import-3.7!skip +plugins/module_utils/client/resource.py import-2.7!skip +plugins/module_utils/client/resource.py import-3.7!skip +plugins/module_utils/client/discovery.py future-import-boilerplate!skip +plugins/module_utils/client/discovery.py metaclass-boilerplate!skip diff --git a/tests/sanity/ignore-2.9.txt b/tests/sanity/ignore-2.9.txt index b2db7a19..83482030 100644 --- a/tests/sanity/ignore-2.9.txt +++ b/tests/sanity/ignore-2.9.txt @@ -6,4 +6,8 @@ molecule/default/roles/helm/files/appversionless-chart/templates/configmap.yaml molecule/default/roles/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip molecule/default/roles/helm/files/test-chart/templates/configmap.yaml yamllint!skip plugins/module_utils/k8sdynamicclient.py import-2.7!skip -plugins/module_utils/k8sdynamicclient.py import-3.7!skip \ No newline at end of file +plugins/module_utils/k8sdynamicclient.py import-3.7!skip +plugins/module_utils/client/discovery.py future-import-boilerplate!skip +plugins/module_utils/client/discovery.py metaclass-boilerplate!skip +tests/unit/module_utils/test_discoverer.py future-import-boilerplate!skip +tests/unit/module_utils/test_discoverer.py metaclass-boilerplate!skip diff --git a/tests/unit/module_utils/test_discoverer.py b/tests/unit/module_utils/test_discoverer.py new file mode 100644 index 00000000..63a8a9f4 --- /dev/null +++ b/tests/unit/module_utils/test_discoverer.py @@ -0,0 +1,143 @@ +# Copyright [2017] [Red Hat, Inc.] +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest + +from kubernetes.client import ApiClient +from kubernetes.dynamic import Resource + +from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import K8SDynamicClient +from ansible_collections.kubernetes.core.plugins.module_utils.client.discovery import LazyDiscoverer +from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import ResourceList + + +@pytest.fixture(scope='module') +def mock_namespace(): + return Resource( + api_version='v1', + kind='Namespace', + name='namespaces', + namespaced=False, + preferred=True, + prefix='api', + shorter_names=['ns'], + shortNames=['ns'], + singularName='namespace', + verbs=['create', 'delete', 'get', 'list', 'patch', 'update', 'watch'] + ) + + +@pytest.fixture(scope='module') +def mock_templates(): + return Resource( + api_version='v1', + kind='Template', + name='templates', + namespaced=True, + preferred=True, + prefix='api', + shorter_names=[], + shortNames=[], + verbs=['create', 'delete', 'get', 'list', 'patch', 'update', 'watch'] + ) + + +@pytest.fixture(scope='module') +def mock_processedtemplates(): + return Resource( + api_version='v1', + kind='Template', + name='processedtemplates', + namespaced=True, + preferred=True, + prefix='api', + shorter_names=[], + shortNames=[], + verbs=['create', 'delete', 'get', 'list', 'patch', 'update', 'watch'] + ) + + +@pytest.fixture(scope='module') +def mock_namespace_list(mock_namespace): + ret = ResourceList(mock_namespace.client, mock_namespace.group, mock_namespace.api_version, mock_namespace.kind) + ret._ResourceList__base_resource = mock_namespace + return ret + + +@pytest.fixture(scope='function', autouse=True) +def setup_client_monkeypatch(monkeypatch, mock_namespace, mock_namespace_list, mock_templates, mock_processedtemplates): + + def mock_load_server_info(self): + self.__version = {'kubernetes': 'mock-k8s-version'} + + def mock_parse_api_groups(self, request_resources=False): + return { + 'api': { + '': { + 'v1': { + 'Namespace': [mock_namespace], + 'NamespaceList': [mock_namespace_list], + 'Template': [mock_templates, mock_processedtemplates], + } + } + } + } + + monkeypatch.setattr(LazyDiscoverer, '_load_server_info', mock_load_server_info) + monkeypatch.setattr(LazyDiscoverer, 'parse_api_groups', mock_parse_api_groups) + + +@pytest.fixture +def client(request): + return K8SDynamicClient(ApiClient(), discoverer=LazyDiscoverer) + + +@pytest.mark.parametrize(("attribute", "value"), [ + ('name', 'namespaces'), + ('singular_name', 'namespace'), + ('short_names', ['ns']) +]) +def test_search_returns_single_and_list(client, mock_namespace, mock_namespace_list, attribute, value): + resources = client.resources.search(**{'api_version': 'v1', attribute: value}) + + assert len(resources) == 2 + assert mock_namespace in resources + assert mock_namespace_list in resources + + +@pytest.mark.parametrize(("attribute", "value"), [ + ('kind', 'Namespace'), + ('name', 'namespaces'), + ('singular_name', 'namespace'), + ('short_names', ['ns']) +]) +def test_get_returns_only_single(client, mock_namespace, attribute, value): + resource = client.resources.get(**{'api_version': 'v1', attribute: value}) + + assert resource == mock_namespace + + +def test_get_namespace_list_kind(client, mock_namespace_list): + resource = client.resources.get(api_version='v1', kind='NamespaceList') + + assert resource == mock_namespace_list + + +def test_search_multiple_resources_for_template(client, mock_templates, mock_processedtemplates): + resources = client.resources.search(api_version='v1', kind='Template') + + assert len(resources) == 2 + assert mock_templates in resources + assert mock_processedtemplates in resources