Merge pull request #690 from barpavel/fix-686-module-utils-deprecation (#734)

Fix all deprecated module_utils imports before ansible-core 2.24 removal

SUMMARY
Fixes all deprecated ansible.module_utils imports across the entire collection that will be removed in ansible-core 2.24.
This PR comprehensively addresses deprecation warnings reported in #686 by updating import statements in 20 files to use the new recommended import paths, and removes 8 unused test utility files that contained deprecated imports.
Deprecated imports replaced:

Deprecated import
Replacement

ansible.module_utils._text
ansible.module_utils.common.text.converters

ansible.module_utils.common._collections_compat
collections.abc

ansible.module_utils.six.moves.shlex_quote
shlex.quote

ansible.module_utils.six.moves.reduce
functools.reduce

ansible.module_utils.six.moves.urllib.parse.urlparse
urllib.parse.urlparse

ansible.module_utils.six.string_types
basestring/str (Python 2/3 compatible)

ansible.module_utils.six.text_type
str

ansible.module_utils.six.PY3
Removed (simplified Python 2/3 conditionals)

ansible.module_utils.six.with_metaclass
Native metaclass= syntax

ansible.module_utils.six.iteritems
dict.items()

Files fixed (20 files, 1 commit per file for easier review):

plugins/action/patch.py
plugins/action/synchronize.py
plugins/callback/cgroup_perf_recap.py
plugins/callback/json.py
plugins/callback/jsonl.py
plugins/callback/profile_roles.py
plugins/callback/profile_tasks.py
plugins/modules/acl.py
plugins/modules/authorized_key.py
plugins/modules/firewalld_info.py
plugins/modules/mount.py
plugins/modules/patch.py
plugins/modules/rhel_rpm_ostree.py
plugins/modules/rpm_ostree_upgrade.py
plugins/modules/seboolean.py
plugins/modules/synchronize.py
plugins/modules/sysctl.py
plugins/shell/csh.py
plugins/shell/fish.py
tests/unit/modules/system/test_mount.py

Files deleted (8 unused test utility files):
These files are dead code - none of them are imported or used anywhere in the test suite or the collection. Removing them also addresses Python 2.7 compatibility concerns raised in code review, as several contained deprecated imports that would be incorrect to fix for Python 2.

tests/unit/compat/builtins.py
tests/unit/mock/loader.py
tests/unit/mock/path.py
tests/unit/mock/procenv.py
tests/unit/mock/vault_helper.py
tests/unit/mock/yaml_helper.py
tests/unit/modules/conftest.py
tests/unit/modules/utils.py

Completeness verified with:
git grep -n -P '_compat|utils._text|utils.six' -- '*.py' | grep -v yml

This command returns no results, confirming all deprecated imports have been replaced.
Notes on Python 2.7 compatibility:
For modules that may run on Python 2.7 managed hosts (e.g., authorized_key.py, synchronize.py, sysctl.py), Python 2/3 compatible fallbacks were used instead of direct Python 3 replacements:

authorized_key.py: try/except ImportError for urllib.parse.urlparse (falls back to urlparse on Python 2)
synchronize.py: try/except ImportError for shlex.quote (falls back to pipes.quote on Python 2)
sysctl.py: uses sys.version_info to set string_types to str on Python 3 (basestring on Python 2)

Also removes corresponding pylint:ansible-bad-import-from entries from tests/sanity/ignore-2.21.txt and tests/sanity/ignore-2.22.txt where applicable.
Fixes #686
ISSUE TYPE

Bugfix Pull Request

ADDITIONAL INFORMATION
Approach:
Each file is fixed in a separate commit for easier code review. The changelog fragment is added in a final commit. Corresponding pylint:ansible-bad-import-from ignore entries in tests/sanity/ignore-2.21.txt and tests/sanity/ignore-2.22.txt are removed in the same commit as the file fix (or the file removal commit).
CI results:
All 59 checks passing (Azure Pipelines sanity, units, lint, Docker, Remote across ansible-core 2.17 through devel, and Zuul ansible/check).

Reviewed-by: Felix Fontein <felix@fontein.de>
Reviewed-by: Pavel Bar
Reviewed-by: Abhijeet Kasurde
(cherry picked from commit 2022c1bd86)

Co-authored-by: centosinfra-prod-github-app[bot] <161850885+centosinfra-prod-github-app[bot]@users.noreply.github.com>

plugins/action/patch.py

@@ -21,7 +21,7 @@ __metaclass__ = type
 import os
 
 from ansible.errors import AnsibleError, AnsibleAction, _AnsibleActionDone, AnsibleActionFail
-from ansible.module_utils._text import to_native
+from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.action import ActionBase
 

plugins/action/synchronize.py

@@ -18,12 +18,11 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os.path
+from collections.abc import MutableSequence
+from shlex import quote as shlex_quote
 
 from ansible import constants as C
-from ansible.module_utils.six import string_types
-from ansible.module_utils.six.moves import shlex_quote
-from ansible.module_utils._text import to_text
-from ansible.module_utils.common._collections_compat import MutableSequence
+from ansible.module_utils.common.text.converters import to_text
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.action import ActionBase
 from ansible.plugins.loader import connection_loader
@@ -417,7 +416,7 @@ class ActionModule(ActionBase):
             # Replicate what we do in the module argumentspec handling for lists
             if not isinstance(_tmp_args.get('rsync_opts'), MutableSequence):
                 tmp_rsync_opts = _tmp_args.get('rsync_opts', [])
-                if isinstance(tmp_rsync_opts, string_types):
+                if isinstance(tmp_rsync_opts, str):
                     tmp_rsync_opts = tmp_rsync_opts.split(',')
                 elif isinstance(tmp_rsync_opts, (int, float)):
                     tmp_rsync_opts = [to_text(tmp_rsync_opts)]

plugins/callback/cgroup_perf_recap.py

@@ -138,8 +138,7 @@ from abc import ABCMeta, abstractmethod
 
 from functools import partial
 
-from ansible.module_utils._text import to_bytes, to_text
-from ansible.module_utils.six import with_metaclass
+from ansible.module_utils.common.text.converters import to_bytes, to_text
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins.callback import CallbackBase
 
@@ -155,7 +154,7 @@ def dict_fromkeys(keys, default=None):
     return d
 
 
-class BaseProf(with_metaclass(ABCMeta, threading.Thread)):
+class BaseProf(threading.Thread, metaclass=ABCMeta):
     def __init__(self, path, obj=None, writer=None):
         threading.Thread.__init__(self)  # pylint: disable=non-parent-init-called
         self.obj = obj

plugins/callback/json.py

@@ -48,7 +48,7 @@ import json
 from functools import partial
 
 from ansible.inventory.host import Host
-from ansible.module_utils._text import to_text
+from ansible.module_utils.common.text.converters import to_text
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins.callback import CallbackBase
 

plugins/callback/jsonl.py

@@ -50,7 +50,7 @@ import copy
 from functools import partial
 
 from ansible.inventory.host import Host
-from ansible.module_utils._text import to_text
+from ansible.module_utils.common.text.converters import to_text
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins.callback import CallbackBase
 

plugins/callback/profile_roles.py

@@ -33,7 +33,7 @@ import collections
 import time
 
 from ansible.plugins.callback import CallbackBase
-from ansible.module_utils.six.moves import reduce
+from functools import reduce
 
 # define start time
 t0 = tn = time.time()

plugins/callback/profile_tasks.py

@@ -86,7 +86,7 @@ import collections
 
 from datetime import datetime
 
-from ansible.module_utils.six.moves import reduce
+from functools import reduce
 from ansible.plugins.callback import CallbackBase
 
 

plugins/modules/acl.py

@@ -147,7 +147,7 @@ import os
 import platform
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_native
+from ansible.module_utils.common.text.converters import to_native
 
 
 def split_entry(entry):

plugins/modules/authorized_key.py

@@ -229,10 +229,17 @@ import errno
 import traceback
 from operator import itemgetter
 
-from ansible.module_utils._text import to_native
+# TODO(Python2): urllib.parse is available in Python 3. This module may run on
+# target hosts with Python 2.7 (e.g., older RHEL systems in CI integration tests).
+# Remove the try/except fallback to urlparse when Python 2 support is dropped.
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
 from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.urls import fetch_url
-from ansible.module_utils.six.moves.urllib.parse import urlparse
 
 
 class keydict(dict):

plugins/modules/firewalld_info.py

@@ -210,7 +210,7 @@ firewalld_info:
 '''
 
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
-from ansible.module_utils._text import to_native
+from ansible.module_utils.common.text.converters import to_native
 from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 from ansible_collections.ansible.posix.plugins.module_utils.version import StrictVersion
 

plugins/modules/mount.py

@@ -225,8 +225,7 @@ import platform
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible_collections.ansible.posix.plugins.module_utils.mount import ismount
-from ansible.module_utils.six import iteritems
-from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common.text.converters import to_bytes, to_native
 from ansible.module_utils.parsing.convert_bool import boolean
 
 
@@ -279,7 +278,7 @@ def _set_mount_save_old(module, args):
     old_lines = []
     exists = False
     changed = False
-    escaped_args = dict([(k, _escape_fstab(v)) for k, v in iteritems(args)])
+    escaped_args = dict([(k, _escape_fstab(v)) for k, v in args.items()])
     new_line = '%(src)s %(name)s %(fstype)s %(opts)s %(dump)s %(passno)s\n'
 
     if platform.system() == 'SunOS':

plugins/modules/patch.py

@@ -106,7 +106,7 @@ import os
 import platform
 from traceback import format_exc
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_native
+from ansible.module_utils.common.text.converters import to_native
 
 
 class PatchError(Exception):

plugins/modules/rhel_rpm_ostree.py

@@ -70,7 +70,7 @@ import os
 import traceback
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_text
+from ansible.module_utils.common.text.converters import to_text
 
 
 def locally_installed(module, pkgname):

plugins/modules/rpm_ostree_upgrade.py

@@ -68,7 +68,7 @@ import os
 import traceback
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common.text.converters import to_native, to_text
 
 
 def rpm_ostree_transaction(module):

plugins/modules/seboolean.py

@@ -73,7 +73,7 @@ except ImportError:
     HAVE_SEMANAGE = False
 
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
-from ansible.module_utils._text import to_text
+from ansible.module_utils.common.text.converters import to_text
 from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 
 

plugins/modules/synchronize.py

@@ -367,9 +367,16 @@ EXAMPLES = r'''
 import os
 import errno
 
+# TODO(Python2): shlex.quote was added in Python 3.3. This module may run on
+# target hosts with Python 2.7 (e.g., older RHEL systems in CI integration tests).
+# Remove the try/except fallback to pipes.quote when Python 2 support is dropped.
+try:
+    from shlex import quote as shlex_quote
+except ImportError:
+    from pipes import quote as shlex_quote
+
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_bytes, to_native
-from ansible.module_utils.six.moves import shlex_quote
+from ansible.module_utils.common.text.converters import to_bytes, to_native
 
 
 client_addr = None

plugins/modules/sysctl.py

@@ -107,12 +107,20 @@ EXAMPLES = r'''
 import os
 import platform
 import re
+import sys
 import tempfile
 
+# TODO(Python2): On Python 2, string_types is basestring (str + unicode).
+# This module may run on target hosts with Python 2.7.
+# Remove the Python 2 branch when Python 2 support is dropped.
+if sys.version_info >= (3, 0):
+    string_types = str
+else:
+    string_types = basestring  # pylint: disable=undefined-variable
+
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.six import string_types
+from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.parsing.convert_bool import BOOLEANS_FALSE, BOOLEANS_TRUE
-from ansible.module_utils._text import to_native
 
 
 class SysctlModule(object):

plugins/shell/csh.py

@@ -13,8 +13,7 @@ DOCUMENTATION = '''
       - shell_common
 '''
 
-from ansible.module_utils.six import text_type
-from ansible.module_utils.six.moves import shlex_quote
+from shlex import quote as shlex_quote
 from ansible.plugins.shell import ShellBase
 
 
@@ -43,5 +42,5 @@ class ShellModule(ShellBase):
         ret = []
         # All the -u options must be first, so we process them first
         ret += ['-u %s' % k for k, v in kwargs.items() if v is None]
-        ret += ['%s=%s' % (k, shlex_quote(text_type(v))) for k, v in kwargs.items() if v is not None]
+        ret += ['%s=%s' % (k, shlex_quote(str(v))) for k, v in kwargs.items() if v is not None]
         return 'env %s' % ' '.join(ret)

plugins/shell/fish.py

@@ -13,8 +13,7 @@ DOCUMENTATION = '''
       - shell_common
 '''
 
-from ansible.module_utils.six import text_type
-from ansible.module_utils.six.moves import shlex_quote
+from shlex import quote as shlex_quote
 from ansible.plugins.shell.sh import ShellModule as ShModule
 
 
@@ -42,7 +41,7 @@ class ShellModule(ShModule):
             if v is None:
                 ret.append('set -e %s;' % k)
             else:
-                ret.append('set -lx %s %s;' % (k, shlex_quote(text_type(v))))
+                ret.append('set -lx %s %s;' % (k, shlex_quote(str(v))))
         return ' '.join(ret)
 
     def build_module_command(self, env_string, shebang, cmd, arg_path=None):