[PR #676/19559892 backport][stable-2.4] fix(Collection's util resource discovery fails when complex subresources present #659) (#688)

* update node image

* fix(Collection's util resource discovery fails when complex subresources present #659)

* fix(add changelog fragment)

* Create discovery.yml

* Update main.yml

* Backport PR659

* fix sanity issues

---------

Co-authored-by: Bastien Bosser <bastien.bosser@eviden.com>

.github/workflows/integration-tests.yaml

@@ -132,6 +132,8 @@ jobs:
 
       - name: create kubernetes cluster
         uses: helm/kind-action@v1.8.0
+        with:
+          node_image: "kindest/node:v1.29.2"
 
       - name: Run integration tests
         uses: ansible-network/github_actions/.github/actions/ansible_test_integration@main

changelogs/fragments/20240222-Collections-util-resource-discovery-fails-when-complex-subresources-present.yml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+  - Resolve Collections util resource discovery fails when complex subresources present (https://github.com/ansible-collections/kubernetes.core/pull/676).

plugins/module_utils/client/discovery.py

@@ -114,7 +114,7 @@ class Discoverer(kubernetes.dynamic.discovery.Discoverer):
             filter(lambda resource: "/" in resource["name"], resources_response)
         )
         for subresource in subresources_raw:
-            resource, name = subresource["name"].split("/")
+            resource, name = subresource["name"].split("/", 1)
             subresources[resource][name] = subresource
 
         for resource in resources_raw:

tests/integration/targets/k8s_info/tasks/discovery.yml

@@ -0,0 +1,32 @@
+---
+# Testing fix for issue https://github.com/ansible-collections/kubernetes.core/pull/676
+- vars:
+    kubevirt_release: "v1.1.1"
+  block:
+    - name: Delete existing namespace
+      kubernetes.core.k8s:
+        kind: namespace
+        namespace: kubevirt
+        state: absent
+
+    - name: Create kubevirt resources
+      kubernetes.core.k8s:
+        state: present
+        apply: true
+        src: "{{ item }}"
+      with_items:
+        - "https://github.com/kubevirt/kubevirt/releases/download/{{ kubevirt_release }}/kubevirt-operator.yaml"
+        - "https://github.com/kubevirt/kubevirt/releases/download/{{ kubevirt_release }}/kubevirt-cr.yaml"
+
+    - name: Read kubevirt Deployment
+      k8s_info:
+        kind: Deployment
+        namespace: kubevirt
+  always:
+    - name: Delete existing namespace
+      kubernetes.core.k8s:
+        kind: namespace
+        namespace: kubevirt
+        state: absent
+        wait: true
+      ignore_errors: true

tests/integration/targets/k8s_info/tasks/main.yml

@@ -3,3 +3,4 @@
   with_items:
     - wait
     - api-server-caching
+    - discovery

tests/sanity/ignore-2.14.txt

@@ -1,9 +1,18 @@
+plugins/module_utils/client/discovery.py import-3.6!skip
+plugins/module_utils/client/discovery.py import-3.7!skip
+plugins/module_utils/client/discovery.py import-3.8!skip
 plugins/module_utils/client/discovery.py import-3.9!skip
 plugins/module_utils/client/discovery.py import-3.10!skip
 plugins/module_utils/client/discovery.py import-3.11!skip
+plugins/module_utils/client/resource.py import-3.6!skip
+plugins/module_utils/client/resource.py import-3.7!skip
+plugins/module_utils/client/resource.py import-3.8!skip
 plugins/module_utils/client/resource.py import-3.9!skip
 plugins/module_utils/client/resource.py import-3.10!skip
 plugins/module_utils/client/resource.py import-3.11!skip
+plugins/module_utils/k8sdynamicclient.py import-3.6!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.8!skip
 plugins/module_utils/k8sdynamicclient.py import-3.9!skip
 plugins/module_utils/k8sdynamicclient.py import-3.10!skip
 plugins/module_utils/k8sdynamicclient.py import-3.11!skip
@@ -19,7 +28,6 @@ tests/integration/targets/helm/files/test-chart-v2/templates/configmap.yaml yaml
 tests/integration/targets/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/helm_diff/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/k8s_scale/files/deployment.yaml yamllint!skip
-tests/sanity/refresh_ignore_files shebang!skip
 plugins/modules/k8s.py validate-modules:return-syntax-error
 plugins/modules/k8s_scale.py validate-modules:return-syntax-error
 plugins/modules/k8s_service.py validate-modules:return-syntax-error

tests/sanity/ignore-2.15.txt

@@ -1,4 +1,3 @@
-plugins/module_utils/version.py pylint!skip
 plugins/module_utils/client/discovery.py import-3.6!skip
 plugins/module_utils/client/discovery.py import-3.7!skip
 plugins/module_utils/client/discovery.py import-3.8!skip
@@ -17,6 +16,7 @@ plugins/module_utils/k8sdynamicclient.py import-3.8!skip
 plugins/module_utils/k8sdynamicclient.py import-3.9!skip
 plugins/module_utils/k8sdynamicclient.py import-3.10!skip
 plugins/module_utils/k8sdynamicclient.py import-3.11!skip
+plugins/module_utils/version.py pylint!skip
 plugins/modules/k8s.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_scale.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_service.py validate-modules:parameter-type-not-in-doc
@@ -29,7 +29,6 @@ tests/integration/targets/helm/files/test-chart-v2/templates/configmap.yaml yaml
 tests/integration/targets/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/helm_diff/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/k8s_scale/files/deployment.yaml yamllint!skip
-tests/sanity/refresh_ignore_files shebang!skip
 plugins/modules/k8s.py validate-modules:return-syntax-error
 plugins/modules/k8s_scale.py validate-modules:return-syntax-error
 plugins/modules/k8s_service.py validate-modules:return-syntax-error

tests/sanity/ignore-2.16.txt

@@ -1,4 +1,3 @@
-plugins/module_utils/version.py pylint!skip
 plugins/module_utils/client/discovery.py import-3.6!skip
 plugins/module_utils/client/discovery.py import-3.7!skip
 plugins/module_utils/client/discovery.py import-3.8!skip
@@ -20,6 +19,7 @@ plugins/module_utils/k8sdynamicclient.py import-3.9!skip
 plugins/module_utils/k8sdynamicclient.py import-3.10!skip
 plugins/module_utils/k8sdynamicclient.py import-3.11!skip
 plugins/module_utils/k8sdynamicclient.py import-3.12!skip
+plugins/module_utils/version.py pylint!skip
 plugins/modules/k8s.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_scale.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_service.py validate-modules:parameter-type-not-in-doc
@@ -32,7 +32,6 @@ tests/integration/targets/helm/files/test-chart-v2/templates/configmap.yaml yaml
 tests/integration/targets/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/helm_diff/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/k8s_scale/files/deployment.yaml yamllint!skip
-tests/sanity/refresh_ignore_files shebang!skip
 plugins/modules/k8s.py validate-modules:return-syntax-error
 plugins/modules/k8s_scale.py validate-modules:return-syntax-error
 plugins/modules/k8s_service.py validate-modules:return-syntax-error

tests/sanity/ignore-2.17.txt

@@ -1,25 +1,16 @@
-plugins/module_utils/version.py pylint!skip
-plugins/module_utils/client/discovery.py import-3.6!skip
-plugins/module_utils/client/discovery.py import-3.7!skip
-plugins/module_utils/client/discovery.py import-3.8!skip
 plugins/module_utils/client/discovery.py import-3.9!skip
 plugins/module_utils/client/discovery.py import-3.10!skip
 plugins/module_utils/client/discovery.py import-3.11!skip
 plugins/module_utils/client/discovery.py import-3.12!skip
-plugins/module_utils/client/resource.py import-3.6!skip
-plugins/module_utils/client/resource.py import-3.7!skip
-plugins/module_utils/client/resource.py import-3.8!skip
 plugins/module_utils/client/resource.py import-3.9!skip
 plugins/module_utils/client/resource.py import-3.10!skip
 plugins/module_utils/client/resource.py import-3.11!skip
 plugins/module_utils/client/resource.py import-3.12!skip
-plugins/module_utils/k8sdynamicclient.py import-3.6!skip
-plugins/module_utils/k8sdynamicclient.py import-3.7!skip
-plugins/module_utils/k8sdynamicclient.py import-3.8!skip
 plugins/module_utils/k8sdynamicclient.py import-3.9!skip
 plugins/module_utils/k8sdynamicclient.py import-3.10!skip
 plugins/module_utils/k8sdynamicclient.py import-3.11!skip
 plugins/module_utils/k8sdynamicclient.py import-3.12!skip
+plugins/module_utils/version.py pylint!skip
 plugins/modules/k8s.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_scale.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_service.py validate-modules:parameter-type-not-in-doc
@@ -32,7 +23,6 @@ tests/integration/targets/helm/files/test-chart-v2/templates/configmap.yaml yaml
 tests/integration/targets/helm/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/helm_diff/files/test-chart/templates/configmap.yaml yamllint!skip
 tests/integration/targets/k8s_scale/files/deployment.yaml yamllint!skip
-tests/sanity/refresh_ignore_files shebang!skip
 plugins/modules/k8s.py validate-modules:return-syntax-error
 plugins/modules/k8s_scale.py validate-modules:return-syntax-error
 plugins/modules/k8s_service.py validate-modules:return-syntax-error

tests/sanity/refresh_ignore_files

@@ -1,216 +0,0 @@
-#!/usr/bin/env python3
-
-
-import itertools
-
-from pathlib import Path
-
-
-# Mapping of Ansible versions to supported Python versions
-ANSIBLE_VERSIONS = {
-    "2.9": ["3.6", "3.7", "3.8"],
-    "2.10": ["3.6", "3.7", "3.8", "3.9"],
-    "2.11": ["3.6", "3.7", "3.8", "3.9"],
-    "2.12": ["3.6", "3.7", "3.8", "3.9", "3.10"],
-    "2.13": ["3.6", "3.7", "3.8", "3.9", "3.10"],
-    "2.14": ["3.6", "3.7", "3.8", "3.9", "3.10"],
-    "2.15": ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"],
-}
-
-IMPORT_SKIPS = [
-    "plugins/module_utils/client/discovery.py",
-    "plugins/module_utils/client/resource.py",
-    "plugins/module_utils/k8sdynamicclient.py",
-]
-
-# Adds validate-modules:parameter-type-not-in-doc
-PARAM_TYPE_SKIPS = [
-    "plugins/modules/k8s.py",
-    "plugins/modules/k8s_scale.py",
-    "plugins/modules/k8s_service.py",
-]
-
-# Adds validate-modules:return-syntax-error
-RETURN_SYNTAX_SKIPS = [
-    "plugins/modules/k8s.py",
-    "plugins/modules/k8s_scale.py",
-    "plugins/modules/k8s_service.py",
-    "plugins/modules/k8s_taint.py",
-]
-
-YAML_LINT_SKIPS = [
-    "tests/unit/module_utils/fixtures/definitions.yml",
-    "tests/unit/module_utils/fixtures/deployments.yml",
-    "tests/unit/module_utils/fixtures/pods.yml",
-    "tests/integration/targets/helm/files/appversionless-chart-v2/templates/configmap.yaml",
-    "tests/integration/targets/helm/files/appversionless-chart/templates/configmap.yaml",
-    "tests/integration/targets/helm/files/test-chart-v2/templates/configmap.yaml",
-    "tests/integration/targets/helm/files/test-chart/templates/configmap.yaml",
-    "tests/integration/targets/helm_diff/files/test-chart/templates/configmap.yaml",
-    "tests/integration/targets/k8s_scale/files/deployment.yaml",
-]
-
-# Add shebang!skip
-SHEBANG_SKIPS = [
-    "tests/sanity/refresh_ignore_files",
-]
-
-# Add validate-modules:import-error
-VALIDATE_IMPORT_SKIPS = [
-    "plugins/modules/k8s.py",
-    "plugins/modules/k8s_cp.py",
-    "plugins/modules/k8s_drain.py",
-    "plugins/modules/k8s_exec.py",
-    "plugins/modules/k8s_info.py",
-    "plugins/modules/k8s_json_patch.py",
-    "plugins/modules/k8s_log.py",
-    "plugins/modules/k8s_rollback.py",
-    "plugins/modules/k8s_scale.py",
-    "plugins/modules/k8s_service.py",
-    "plugins/modules/k8s_taint.py",
-]
-
-
-def import_skips(*versions):
-    for f in IMPORT_SKIPS:
-        for v in versions:
-            yield f"{f} import-{v}!skip"
-
-def param_type_skips(ansible_version):
-    if ansible_version not in ("2.9", "2.10"):
-        for f in PARAM_TYPE_SKIPS:
-            yield f"{f} validate-modules:parameter-type-not-in-doc"
-
-
-def return_syntax_skips(ansible_version):
-    if ansible_version not in ("2.9", "2.10"):
-        for f in RETURN_SYNTAX_SKIPS:
-            yield f"{f} validate-modules:return-syntax-error"
-    else:
-        yield
-
-
-def yaml_lint_skips():
-    for f in YAML_LINT_SKIPS:
-        yield f"{f} yamllint!skip"
-
-
-def shebang_skips():
-    for f in SHEBANG_SKIPS:
-        yield f"{f} shebang!skip"
-
-
-def import_boilerplate(path, ansible_version):
-    if ansible_version in ("2.9", "2.10", "2.11"):
-        for f in (p for p in path.glob("**/*.py") if not p.is_symlink()):
-            yield f"{f} future-import-boilerplate!skip"
-    else:
-        yield
-
-
-def metaclass_boilerplate(path, ansible_version):
-    if ansible_version in ("2.9", "2.10", "2.11"):
-        for f in (p for p in path.glob("**/*.py") if not p.is_symlink()):
-            yield f"{f} metaclass-boilerplate!skip"
-    else:
-        yield
-
-
-def unsupported_compile_skips(path, ansible_version):
-    """This adds rules for compile skips for all unsupported versions of python.
-
-    These aren't needed for Ansible version 2.12+ as that can be managed on a
-    global level in tests/config.yml.
-    """
-    if ansible_version in ("2.9", "2.10", "2.11"):
-        for f in (p for p in path.glob("**/*.py") if not p.is_symlink()):
-            yield (
-                f"{f} compile-2.6!skip\n"
-                f"{f} compile-2.7!skip\n"
-                f"{f} compile-3.5!skip"
-            )
-
-
-def unsupported_import_skips(path, ansible_version):
-    """This adds rules for import skips for all unsupported versions of python.
-
-    These aren't needed for Ansible version 2.12+ as that can be managed on a
-    global level in tests/config.yml.
-    """
-    if ansible_version in ("2.9", "2.10", "2.11"):
-        if ansible_version in ("2.9", "2.10") and path.name == "plugins":
-            pathglob = itertools.chain(
-                path.joinpath("modules").glob("**/*.py"),
-                path.joinpath("module_utils").glob("**/*.py")
-            )
-        else:
-            pathglob = path.glob("**/*.py")
-        for f in (p for p in pathglob if not p.is_symlink()):
-            yield (
-                f"{f} import-2.6!skip\n"
-                f"{f} import-2.7!skip\n"
-                f"{f} import-3.5!skip"
-            )
-
-
-def unsupported_pylint_skips(path, ansible_version):
-    """This adds rules to skip pylint checks.
-
-    This is only a problem on Ansible version 2.9 and 2.10 with python 3.5,
-    but there's no way to restrict this to a specific version of python.
-    """
-    if ansible_version in ("2.9", "2.10"):
-        pathglob = itertools.chain(
-            path.joinpath("plugins/modules").glob("**/*.py"),
-            path.joinpath("plugins/module_utils").glob("**/*.py"),
-            path.joinpath("tests").glob("**/*.py"),
-        )
-        for f in (p for p in pathglob if not p.is_symlink()):
-            yield f"{f} pylint!skip"
-
-
-def unsupported_validate_modules_skips(ansible_version):
-    """Disable validate-modules test.
-
-    Unfortunately, this is overly broad. Applying a validate-modules:import-error
-    skip fixes ansible 2.9 and python <3.6, but causes validation of the ignores
-    file itself to fail in python 3.6+. The only solution here is to simply
-    skip validate-modules altogether.
-    """
-    if ansible_version in ("2.9", "2.10"):
-        for f in VALIDATE_IMPORT_SKIPS:
-            yield f"{f} validate-modules!skip"
-
-
-def main():
-    target_dir = Path('.')
-    sanity_dir = target_dir / "tests" / "sanity"
-    plugins = target_dir / "plugins"
-    units = target_dir / "tests" / "unit"
-    integration = target_dir / "tests" / "integration"
-
-    for ansible, python in ANSIBLE_VERSIONS.items():
-        with open(sanity_dir / f"ignore-{ansible}.txt", "w") as fp:
-            ignores = itertools.chain(
-                import_skips(*python),
-                param_type_skips(ansible),
-                yaml_lint_skips(),
-                shebang_skips(),
-                return_syntax_skips(ansible),
-                import_boilerplate(plugins, ansible),
-                import_boilerplate(units, ansible),
-                metaclass_boilerplate(plugins, ansible),
-                metaclass_boilerplate(units, ansible),
-                unsupported_import_skips(plugins, ansible),
-                unsupported_compile_skips(plugins, ansible),
-                unsupported_compile_skips(units, ansible),
-                unsupported_compile_skips(integration, ansible),
-                unsupported_pylint_skips(target_dir, ansible),
-                unsupported_validate_modules_skips(ansible),
-            )
-            for f in filter(None, ignores):
-                fp.write(f + "\n")
-
-
-if __name__ == "__main__":
-    main()