Cleanup GitHub workflows (#655)

* Cleanup gha

* test by removing matrix excludes

* Rename sanity tests

* trigger integration tests

* Fix ansible-lint workflow

* Fix concurrency

* Add ansible-lint config

* Add ansible-lint config

* Fix integration and lint issues

* integration wf

* fix yamllint issues

* fix yamllint issues

* update readme and add ignore-2.16.txt

* fix ansible-doc

* Add version

* Use /dev/random to generate random data

The GHA environment has difficultly generating entropy. Trying to read
from /dev/urandom just blocks forever. We don't care if the random data
is cryptographically secure; it's just garbage data for the test. Read
from /dev/random, instead. This is only used during the k8s_copy test
target.

This also removes the custom test module that was being used to generate
the files. It's not worth maintaining this for two task that can be
replaced with some simple command/shell tasks.

* Fix saniry errors

* test github_action fix

* Address review comments

* Remove default types

* review comments

* isort fixes

* remove tags

* Add setuptools to venv

* Test gh changes

* update changelog

* update ignore-2.16

* Fix indentation in inventory plugin example

* Update .github/workflows/integration-tests.yaml

* Update integration-tests.yaml

---------

Co-authored-by: Mike Graves <mgraves@redhat.com>
Co-authored-by: Bikouo Aubin <79859644+abikouo@users.noreply.github.com>

tests/integration/targets/helm/library/helm_test_pending.py

@@ -48,9 +48,10 @@ EXAMPLES = r"""
 RETURN = r"""
 """
 
-import subprocess
 import json
+import subprocess
 import time
+
 from ansible.module_utils.basic import AnsibleModule
 
 

tests/integration/targets/helm/library/helm_test_version.py

@@ -52,12 +52,12 @@ result:
 """
 
 import re
+
+from ansible.module_utils.basic import AnsibleModule
 from ansible_collections.kubernetes.core.plugins.module_utils.version import (
     LooseVersion,
 )
 
-from ansible.module_utils.basic import AnsibleModule
-
 
 def main():
     module = AnsibleModule(

tests/integration/targets/k8s_copy/library/k8s_create_file.py

@@ -1,93 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-# Copyright: (c) 2021, Aubin Bikouo <@abikouo>
-# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-
-from __future__ import absolute_import, division, print_function
-
-__metaclass__ = type
-
-
-DOCUMENTATION = r"""
-
-module: k8s_create_file
-
-short_description: Create large file with a defined size.
-
-author:
-    - Aubin Bikouo (@abikouo)
-
-description:
-  - This module is used to validate k8s_cp module.
-
-options:
-  path:
-    description:
-    - The destination path for the file to create.
-    type: path
-    required: yes
-  size:
-    description:
-    - The size of the output file in MB.
-    type: int
-    default: 400
-  binary:
-    description:
-    - If this flag is set to yes, the generated file content binary data.
-    type: bool
-    default: False
-"""
-
-EXAMPLES = r"""
-- name: create 150MB file
-  k8s_diff:
-    path: large_file.txt
-    size: 150
-"""
-
-
-RETURN = r"""
-"""
-
-import os
-
-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_native
-
-
-def execute_module(module):
-    try:
-        size = module.params.get("size") * 1024 * 1024
-        path = module.params.get("path")
-        write_mode = "w"
-        if module.params.get("binary"):
-            content = os.urandom(size)
-            write_mode = "wb"
-        else:
-            content = ""
-            count = 0
-            while len(content) < size:
-                content += "This file has been generated using ansible: {0}\n".format(
-                    count
-                )
-                count += 1
-
-        with open(path, write_mode) as f:
-            f.write(content)
-        module.exit_json(changed=True, size=len(content))
-    except Exception as e:
-        module.fail_json(msg="failed to create file due to: {0}".format(to_native(e)))
-
-
-def main():
-    argument_spec = {}
-    argument_spec["size"] = {"type": "int", "default": 400}
-    argument_spec["path"] = {"type": "path", "required": True}
-    argument_spec["binary"] = {"type": "bool", "default": False}
-    module = AnsibleModule(argument_spec=argument_spec)
-
-    execute_module(module)
-
-
-if __name__ == "__main__":
-    main()

tests/integration/targets/k8s_copy/library/kubectl_file_compare.py

@@ -89,10 +89,10 @@ EXAMPLES = r"""
 RETURN = r"""
 """
 
-import os
 import filecmp
-
+import os
 from tempfile import NamedTemporaryFile, TemporaryDirectory
+
 from ansible.module_utils.basic import AnsibleModule
 
 
@@ -157,7 +157,6 @@ def compare_directories(dir1, dir2):
 
 
 def execute_module(module):
-
     args = module.params.get("args")
     local_path = module.params.get("local_path")
     namespace = module.params.get("namespace")

tests/integration/targets/k8s_copy/tasks/test_copy_large_file.yml

@@ -10,16 +10,13 @@
       path: "{{ test_directory }}"
       state: directory
 
-  - name: create large text file
-    k8s_create_file:
-      path: "{{ test_directory }}/large_text_file.txt"
-      size: 150
+  - name: Create a large text file
+    ansible.builtin.shell:
+      cmd: base64 /dev/random | head -c 150M > {{ test_directory }}/large_text_file.txt
 
-  - name: create large binary file
-    k8s_create_file:
-      path: "{{ test_directory }}/large_bin_file.bin"
-      size: 200
-      binary: true
+  - name: Create a large binary file
+    ansible.builtin.command:
+      cmd: dd if=/dev/random of={{ test_directory }}/large_bin_file.bin bs=1M count=200
 
   # Copy large text file from/to local filesystem to Pod
   - name: copy large file into remote Pod

tests/integration/targets/k8s_validate/tasks/main.yml

@@ -52,6 +52,7 @@
         name:
           - kubernetes
           - kubernetes-validate
+          - setuptools
         virtualenv: "{{ virtualenv }}"
         virtualenv_command: "{{ virtualenv_command }}"
         virtualenv_site_packages: false

tests/integration/targets/setup_kubeconfig/library/test_inventory_read_credentials.py

@@ -78,7 +78,6 @@ from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import
 
 class K8SInventoryTestModule(AnsibleModule):
     def __init__(self):
-
         argument_spec = dict(
             kube_config=dict(required=True, type="path"),
             dest_dir=dict(required=True, type="path"),
@@ -88,7 +87,6 @@ class K8SInventoryTestModule(AnsibleModule):
         self.execute_module()
 
     def execute_module(self):
-
         dest_dir = os.path.abspath(self.params.get("dest_dir"))
         kubeconfig_path = self.params.get("kube_config")
         if not os.path.isdir(dest_dir):

tests/sanity/ignore-2.14.txt

@@ -3,16 +3,19 @@ plugins/module_utils/client/discovery.py import-3.7!skip
 plugins/module_utils/client/discovery.py import-3.8!skip
 plugins/module_utils/client/discovery.py import-3.9!skip
 plugins/module_utils/client/discovery.py import-3.10!skip
+plugins/module_utils/client/discovery.py import-3.11!skip
 plugins/module_utils/client/resource.py import-3.6!skip
 plugins/module_utils/client/resource.py import-3.7!skip
 plugins/module_utils/client/resource.py import-3.8!skip
 plugins/module_utils/client/resource.py import-3.9!skip
 plugins/module_utils/client/resource.py import-3.10!skip
+plugins/module_utils/client/resource.py import-3.11!skip
 plugins/module_utils/k8sdynamicclient.py import-3.6!skip
 plugins/module_utils/k8sdynamicclient.py import-3.7!skip
 plugins/module_utils/k8sdynamicclient.py import-3.8!skip
 plugins/module_utils/k8sdynamicclient.py import-3.9!skip
 plugins/module_utils/k8sdynamicclient.py import-3.10!skip
+plugins/module_utils/k8sdynamicclient.py import-3.11!skip
 plugins/modules/k8s.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_scale.py validate-modules:parameter-type-not-in-doc
 plugins/modules/k8s_service.py validate-modules:parameter-type-not-in-doc

tests/sanity/ignore-2.16.txt

@@ -0,0 +1,40 @@
+plugins/module_utils/client/discovery.py import-3.6!skip
+plugins/module_utils/client/discovery.py import-3.7!skip
+plugins/module_utils/client/discovery.py import-3.8!skip
+plugins/module_utils/client/discovery.py import-3.9!skip
+plugins/module_utils/client/discovery.py import-3.10!skip
+plugins/module_utils/client/discovery.py import-3.11!skip
+plugins/module_utils/client/discovery.py import-3.12!skip
+plugins/module_utils/client/resource.py import-3.6!skip
+plugins/module_utils/client/resource.py import-3.7!skip
+plugins/module_utils/client/resource.py import-3.8!skip
+plugins/module_utils/client/resource.py import-3.9!skip
+plugins/module_utils/client/resource.py import-3.10!skip
+plugins/module_utils/client/resource.py import-3.11!skip
+plugins/module_utils/client/resource.py import-3.12!skip
+plugins/module_utils/k8sdynamicclient.py import-3.6!skip
+plugins/module_utils/k8sdynamicclient.py import-3.7!skip
+plugins/module_utils/k8sdynamicclient.py import-3.8!skip
+plugins/module_utils/k8sdynamicclient.py import-3.9!skip
+plugins/module_utils/k8sdynamicclient.py import-3.10!skip
+plugins/module_utils/k8sdynamicclient.py import-3.11!skip
+plugins/module_utils/k8sdynamicclient.py import-3.12!skip
+plugins/module_utils/version.py pylint!skip
+plugins/modules/k8s.py validate-modules:parameter-type-not-in-doc
+plugins/modules/k8s_scale.py validate-modules:parameter-type-not-in-doc
+plugins/modules/k8s_service.py validate-modules:parameter-type-not-in-doc
+tests/unit/module_utils/fixtures/definitions.yml yamllint!skip
+tests/unit/module_utils/fixtures/deployments.yml yamllint!skip
+tests/integration/targets/k8s_delete/files/deployments.yaml yamllint!skip
+tests/unit/module_utils/fixtures/pods.yml yamllint!skip
+tests/integration/targets/helm/files/appversionless-chart-v2/templates/configmap.yaml yamllint!skip
+tests/integration/targets/helm/files/appversionless-chart/templates/configmap.yaml yamllint!skip
+tests/integration/targets/helm/files/test-chart-v2/templates/configmap.yaml yamllint!skip
+tests/integration/targets/helm/files/test-chart/templates/configmap.yaml yamllint!skip
+tests/integration/targets/helm_diff/files/test-chart/templates/configmap.yaml yamllint!skip
+tests/integration/targets/k8s_scale/files/deployment.yaml yamllint!skip
+tests/sanity/refresh_ignore_files shebang!skip
+plugins/modules/k8s.py validate-modules:return-syntax-error
+plugins/modules/k8s_scale.py validate-modules:return-syntax-error
+plugins/modules/k8s_service.py validate-modules:return-syntax-error
+plugins/modules/k8s_taint.py validate-modules:return-syntax-error

tests/sanity/ignore-2.9.txt

@@ -523,9 +523,6 @@ tests/unit/modules/test_module_helm.py compile-3.5!skip
 tests/unit/action/test_remove_omit.py compile-2.6!skip
 tests/unit/action/test_remove_omit.py compile-2.7!skip
 tests/unit/action/test_remove_omit.py compile-3.5!skip
-tests/integration/targets/k8s_copy/library/k8s_create_file.py compile-2.6!skip
-tests/integration/targets/k8s_copy/library/k8s_create_file.py compile-2.7!skip
-tests/integration/targets/k8s_copy/library/k8s_create_file.py compile-3.5!skip
 tests/integration/targets/k8s_copy/library/kubectl_file_compare.py compile-2.6!skip
 tests/integration/targets/k8s_copy/library/kubectl_file_compare.py compile-2.7!skip
 tests/integration/targets/k8s_copy/library/kubectl_file_compare.py compile-3.5!skip
@@ -577,7 +574,6 @@ plugins/module_utils/k8s/client.py pylint!skip
 plugins/module_utils/k8s/runner.py pylint!skip
 plugins/module_utils/k8s/service.py pylint!skip
 plugins/module_utils/k8s/exceptions.py pylint!skip
-tests/integration/targets/k8s_copy/library/k8s_create_file.py pylint!skip
 tests/integration/targets/k8s_copy/library/kubectl_file_compare.py pylint!skip
 tests/integration/targets/setup_kubeconfig/library/test_inventory_read_credentials.py pylint!skip
 tests/integration/targets/helm/library/helm_test_version.py pylint!skip

tests/sanity/refresh_ignore_files

@@ -2,10 +2,8 @@
 
 
 import itertools
-
 from pathlib import Path
 
-
 # Mapping of Ansible versions to supported Python versions
 ANSIBLE_VERSIONS = {
     "2.9": ["3.6", "3.7", "3.8"],

tests/unit/action/test_remove_omit.py

@@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 from datetime import datetime
+
 from ansible_collections.kubernetes.core.plugins.action.k8s_info import RemoveOmit
 
 

tests/unit/conftest.py

@@ -6,12 +6,11 @@ import json
 import sys
 from io import BytesIO
 
-import pytest
-
 import ansible.module_utils.basic
-from ansible.module_utils.six import string_types
+import pytest
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.common._collections_compat import MutableMapping
+from ansible.module_utils.six import string_types
 
 
 @pytest.fixture

tests/unit/module_utils/test_apply.py

@@ -18,8 +18,8 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 from ansible_collections.kubernetes.core.plugins.module_utils.apply import (
-    merge,
     apply_patch,
+    merge,
 )
 
 tests = [

tests/unit/module_utils/test_client.py

@@ -1,14 +1,14 @@
-import os
 import base64
+import os
 import tempfile
-import yaml
-import mock
-from mock import MagicMock
 
+import mock
+import yaml
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import (
     _create_auth_spec,
     _create_configuration,
 )
+from mock import MagicMock
 
 TEST_HOST = "test-host"
 TEST_SSL_HOST = "https://test-host"

tests/unit/module_utils/test_core.py

@@ -6,7 +6,6 @@ import json
 
 import kubernetes
 import pytest
-
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.core import (
     AnsibleK8SModule,
 )

tests/unit/module_utils/test_discoverer.py

@@ -14,19 +14,17 @@
 
 
 import pytest
-
-from kubernetes.client import ApiClient
-from kubernetes.dynamic import Resource
-
-from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import (
-    K8SDynamicClient,
-)
 from ansible_collections.kubernetes.core.plugins.module_utils.client.discovery import (
     LazyDiscoverer,
 )
 from ansible_collections.kubernetes.core.plugins.module_utils.client.resource import (
     ResourceList,
 )
+from ansible_collections.kubernetes.core.plugins.module_utils.k8sdynamicclient import (
+    K8SDynamicClient,
+)
+from kubernetes.client import ApiClient
+from kubernetes.dynamic import Resource
 
 
 @pytest.fixture(scope="module")

tests/unit/module_utils/test_helm.py

@@ -7,18 +7,17 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import os.path
-import yaml
+import random
+import string
 import tempfile
+from unittest.mock import MagicMock
+
 import pytest
-
-
+import yaml
 from ansible_collections.kubernetes.core.plugins.module_utils.helm import (
     AnsibleHelmModule,
     write_temp_kubeconfig,
 )
-from unittest.mock import MagicMock
-import random
-import string
 
 
 @pytest.fixture()
@@ -114,7 +113,6 @@ def test_write_temp_kubeconfig_with_kubeconfig():
 
 
 def test_module_get_helm_binary_from_params():
-
     helm_binary_path = MagicMock()
     helm_sys_binary_path = MagicMock()
 
@@ -129,7 +127,6 @@ def test_module_get_helm_binary_from_params():
 
 
 def test_module_get_helm_binary_from_system():
-
     helm_sys_binary_path = MagicMock()
     module = MagicMock()
     module.params = {}
@@ -140,7 +137,6 @@ def test_module_get_helm_binary_from_system():
 
 
 def test_module_get_helm_plugin_list(_ansible_helm_module):
-
     _ansible_helm_module.run_helm_command = MagicMock()
     _ansible_helm_module.run_helm_command.return_value = (0, "output", "error")
 
@@ -156,7 +152,6 @@ def test_module_get_helm_plugin_list(_ansible_helm_module):
 
 
 def test_module_get_helm_plugin_list_failure(_ansible_helm_module):
-
     _ansible_helm_module.run_helm_command = MagicMock()
     _ansible_helm_module.run_helm_command.return_value = (-1, "output", "error")
 
@@ -175,7 +170,6 @@ def test_module_get_helm_plugin_list_failure(_ansible_helm_module):
 @pytest.mark.parametrize("no_values", [True, False])
 @pytest.mark.parametrize("get_all", [True, False])
 def test_module_get_values(_ansible_helm_module, no_values, get_all):
-
     expected = {"test": "units"}
     output = "---\ntest: units\n"
 
@@ -211,7 +205,6 @@ def test_module_get_values(_ansible_helm_module, no_values, get_all):
     ],
 )
 def test_module_get_helm_version(_ansible_helm_module, output, expected):
-
     _ansible_helm_module.run_command = MagicMock()
     _ansible_helm_module.run_command.return_value = (0, output, "error")
 
@@ -224,7 +217,6 @@ def test_module_get_helm_version(_ansible_helm_module, output, expected):
 
 
 def test_module_run_helm_command(_ansible_helm_module):
-
     error = "".join(
         random.choice(string.ascii_letters + string.digits) for x in range(10)
     )
@@ -252,7 +244,6 @@ def test_module_run_helm_command(_ansible_helm_module):
 
 @pytest.mark.parametrize("fails_on_error", [True, False])
 def test_module_run_helm_command_failure(_ansible_helm_module, fails_on_error):
-
     error = "".join(
         random.choice(string.ascii_letters + string.digits) for x in range(10)
     )
@@ -311,7 +302,6 @@ def test_module_run_helm_command_failure(_ansible_helm_module, fails_on_error):
     ],
 )
 def test_module_prepare_helm_environment(params, env_update, kubeconfig):
-
     module = MagicMock()
     module.params = params
 
@@ -355,7 +345,6 @@ def test_module_prepare_helm_environment(params, env_update, kubeconfig):
 def test_module_prepare_helm_environment_with_validate_certs(
     helm_version, is_env_var_set
 ):
-
     module = MagicMock()
     module.params = {"validate_certs": False}
 
@@ -387,7 +376,6 @@ def test_module_prepare_helm_environment_with_validate_certs(
     ],
 )
 def test_module_prepare_helm_environment_with_ca_cert(helm_version, is_env_var_set):
-
     ca_cert = "".join(
         random.choice(string.ascii_letters + string.digits) for i in range(50)
     )
@@ -441,7 +429,6 @@ def test_module_prepare_helm_environment_with_ca_cert(helm_version, is_env_var_s
     ],
 )
 def test_module_get_helm_set_values_args(set_values, expected):
-
     module = MagicMock()
     module.params = {}
     module.fail_json.side_effect = SystemExit(1)

tests/unit/module_utils/test_runner.py

@@ -1,12 +1,11 @@
-import pytest
 from copy import deepcopy
 from unittest.mock import Mock
 
-from kubernetes.dynamic.resource import ResourceInstance
-
+import pytest
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.runner import (
     perform_action,
 )
+from kubernetes.dynamic.resource import ResourceInstance
 
 definition = {
     "apiVersion": "v1",

tests/unit/module_utils/test_service.py

@@ -1,14 +1,12 @@
 from unittest.mock import Mock
 
 import pytest
-from kubernetes.dynamic.resource import ResourceInstance, Resource
-
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.service import (
     K8sService,
     diff_objects,
 )
-
 from kubernetes.dynamic.exceptions import NotFoundError
+from kubernetes.dynamic.resource import Resource, ResourceInstance
 
 pod_definition = {
     "apiVersion": "v1",

tests/unit/module_utils/test_waiter.py

@@ -5,20 +5,19 @@ from unittest.mock import Mock
 
 import pytest
 import yaml
-from kubernetes.dynamic.resource import ResourceInstance
-from kubernetes.dynamic.exceptions import NotFoundError
-
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.waiter import (
+    DummyWaiter,
+    Waiter,
     clock,
     custom_condition,
     deployment_ready,
-    DummyWaiter,
     exists,
     get_waiter,
     pod_ready,
     resource_absent,
-    Waiter,
 )
+from kubernetes.dynamic.exceptions import NotFoundError
+from kubernetes.dynamic.resource import ResourceInstance
 
 
 def resources(filepath):

tests/unit/modules/test_helm_template_module.py

@@ -7,14 +7,13 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import unittest
-
 from unittest.mock import patch
 
 from ansible.module_utils import basic
 from ansible_collections.kubernetes.core.plugins.modules import helm_template
 from ansible_collections.kubernetes.core.tests.unit.utils.ansible_module_mock import (
-    AnsibleFailJson,
     AnsibleExitJson,
+    AnsibleFailJson,
     exit_json,
     fail_json,
     get_bin_path,

tests/unit/modules/test_module_helm.py

@@ -7,14 +7,13 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import unittest
-
-from unittest.mock import MagicMock, patch, call
+from unittest.mock import MagicMock, call, patch
 
 from ansible.module_utils import basic
 from ansible_collections.kubernetes.core.plugins.modules import helm
 from ansible_collections.kubernetes.core.tests.unit.utils.ansible_module_mock import (
-    AnsibleFailJson,
     AnsibleExitJson,
+    AnsibleFailJson,
     exit_json,
     fail_json,
     get_bin_path,