Python 3.8 collections compatibility fixes.

Includes a new pylint blacklist plugin to prevent regressions.

lib/ansible/plugins/action/synchronize.py

@@ -18,11 +18,11 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os.path
-from collections import MutableSequence
 
 from ansible import constants as C
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import MutableSequence
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.action import ActionBase
 from ansible.plugins.loader import connection_loader

lib/ansible/plugins/cache/__init__.py

@@ -21,12 +21,12 @@ import os
 import time
 import errno
 from abc import ABCMeta, abstractmethod
-from collections import MutableMapping
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import with_metaclass
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.loader import cache_loader
 
 try:

lib/ansible/plugins/cache/memcached.py

@@ -50,6 +50,7 @@ from itertools import chain
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
+from ansible.module_utils.common._collections_compat import MutableSet
 from ansible.plugins.cache import BaseCacheModule
 
 try:
@@ -126,7 +127,7 @@ class ProxyClientPool(object):
             self.release_connection(conn)
 
 
-class CacheModuleKeys(collections.MutableSet):
+class CacheModuleKeys(MutableSet):
     """
     A set subclass that keeps track of insertion time and persists
     the set in memcached.

lib/ansible/plugins/callback/__init__.py

@@ -26,9 +26,9 @@ import sys
 import warnings
 
 from copy import deepcopy
-from collections import MutableMapping
 
 from ansible import constants as C
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins import AnsiblePlugin, get_plugin_class
 from ansible.utils.color import stringc

lib/ansible/plugins/callback/dense.py

@@ -20,8 +20,6 @@ requirements:
 - set as stdout in configuation
 '''
 
-from collections import MutableMapping, MutableSequence
-
 HAS_OD = False
 try:
     from collections import OrderedDict
@@ -30,6 +28,7 @@ except ImportError:
     pass
 
 from ansible.module_utils.six import binary_type, text_type
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
 from ansible.plugins.callback.default import CallbackModule as CallbackModule_default
 from ansible.utils.color import colorize, hostcolor
 

lib/ansible/plugins/callback/log_plays.py

@@ -21,9 +21,9 @@ DOCUMENTATION = '''
 import os
 import time
 import json
-from collections import MutableMapping
 
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.callback import CallbackBase
 
 

lib/ansible/plugins/cliconf/eos.py

@@ -41,12 +41,12 @@ options:
         version_added: '2.7'
 """
 
-import collections
 import json
 import time
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.utils import to_list
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.plugins.cliconf import CliconfBase, enable_mode
@@ -116,7 +116,7 @@ class Cliconf(CliconfBase):
         requests = []
         multiline = False
         for line in to_list(candidate):
-            if not isinstance(line, collections.Mapping):
+            if not isinstance(line, Mapping):
                 line = {'command': line}
 
             cmd = line['command']
@@ -175,7 +175,7 @@ class Cliconf(CliconfBase):
             raise ValueError("'commands' value is required")
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/ios.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import time
 import json
@@ -28,6 +27,7 @@ from itertools import chain
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
@@ -135,7 +135,7 @@ class Cliconf(CliconfBase):
         if commit:
             self.send_command('configure terminal')
             for line in to_list(candidate):
-                if not isinstance(line, collections.Mapping):
+                if not isinstance(line, Mapping):
                     line = {'command': line}
 
                 cmd = line['command']
@@ -279,7 +279,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/iosxr.py

@@ -19,12 +19,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import json
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.connection import ConnectionError
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
@@ -99,7 +99,7 @@ class Cliconf(CliconfBase):
             candidate = 'load {0}'.format(replace)
 
         for line in to_list(candidate):
-            if not isinstance(line, collections.Mapping):
+            if not isinstance(line, Mapping):
                 line = {'command': line}
             cmd = line['command']
             results.append(self.send_command(**line))
@@ -177,7 +177,7 @@ class Cliconf(CliconfBase):
             raise ValueError("'commands' value is required")
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/junos.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import json
 import re
 
@@ -27,6 +26,7 @@ from itertools import chain
 from functools import wraps
 
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase
 
@@ -100,7 +100,7 @@ class Cliconf(CliconfBase):
             candidate = 'load replace {0}'.format(replace)
 
         for line in to_list(candidate):
-            if not isinstance(line, collections.Mapping):
+            if not isinstance(line, Mapping):
                 line = {'command': line}
             cmd = line['command']
             results.append(self.send_command(**line))

lib/ansible/plugins/cliconf/nxos.py

@@ -19,12 +19,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import json
 import re
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.connection import ConnectionError
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
@@ -177,7 +177,7 @@ class Cliconf(CliconfBase):
             self.send_command('configure terminal')
 
             for line in to_list(candidate):
-                if not isinstance(line, collections.Mapping):
+                if not isinstance(line, Mapping):
                     line = {'command': line}
 
                 cmd = line['command']
@@ -204,7 +204,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/voss.py

@@ -19,12 +19,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import json
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase, enable_mode
@@ -126,7 +126,7 @@ class Cliconf(CliconfBase):
         if commit:
             self.send_command('configure terminal')
             for line in to_list(candidate):
-                if not isinstance(line, collections.Mapping):
+                if not isinstance(line, Mapping):
                     line = {'command': line}
 
                 cmd = line['command']
@@ -209,7 +209,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/vyos.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import json
 
@@ -27,6 +26,7 @@ from itertools import chain
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase
@@ -75,7 +75,7 @@ class Cliconf(CliconfBase):
         requests = []
         self.send_command('configure')
         for cmd in to_list(candidate):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             results.append(self.send_command(**cmd))
@@ -199,7 +199,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/filter/core.py

@@ -34,7 +34,6 @@ import time
 import uuid
 import yaml
 
-from collections import MutableMapping
 import datetime
 from functools import partial
 from random import Random, SystemRandom, shuffle, random
@@ -46,6 +45,7 @@ from ansible.module_utils.six import iteritems, string_types, integer_types, rer
 from ansible.module_utils.six.moves import reduce, shlex_quote
 from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.common.collections import is_sequence
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.parsing.yaml.dumper import AnsibleDumper
 from ansible.utils.encrypt import passlib_or_crypt

lib/ansible/plugins/filter/mathstuff.py

@@ -23,7 +23,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 
-import collections
 import itertools
 import math
 
@@ -33,6 +32,7 @@ from ansible.errors import AnsibleFilterError
 from ansible.module_utils import basic
 from ansible.module_utils.six import binary_type, text_type
 from ansible.module_utils.six.moves import zip, zip_longest
+from ansible.module_utils.common._collections_compat import Hashable, Mapping, Iterable
 from ansible.module_utils._text import to_native, to_text
 
 try:
@@ -55,7 +55,7 @@ def unique(environment, a, case_sensitive=False, attribute=None):
     try:
         if HAS_UNIQUE:
             c = do_unique(environment, a, case_sensitive=case_sensitive, attribute=attribute)
-            if isinstance(a, collections.Hashable):
+            if isinstance(a, Hashable):
                 c = set(c)
             else:
                 c = list(c)
@@ -75,7 +75,7 @@ def unique(environment, a, case_sensitive=False, attribute=None):
             raise AnsibleFilterError("Ansible's unique filter does not support case_sensitive nor attribute parameters, "
                                      "you need a newer version of Jinja2 that provides their version of the filter.")
 
-        if isinstance(a, collections.Hashable):
+        if isinstance(a, Hashable):
             c = set(a)
         else:
             c = []
@@ -87,7 +87,7 @@ def unique(environment, a, case_sensitive=False, attribute=None):
 
 @environmentfilter
 def intersect(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) & set(b)
     else:
         c = unique(environment, [x for x in a if x in b])
@@ -96,7 +96,7 @@ def intersect(environment, a, b):
 
 @environmentfilter
 def difference(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) - set(b)
     else:
         c = unique(environment, [x for x in a if x not in b])
@@ -105,7 +105,7 @@ def difference(environment, a, b):
 
 @environmentfilter
 def symmetric_difference(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) ^ set(b)
     else:
         isect = intersect(environment, a, b)
@@ -115,7 +115,7 @@ def symmetric_difference(environment, a, b):
 
 @environmentfilter
 def union(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) | set(b)
     else:
         c = unique(environment, a + b)
@@ -189,15 +189,15 @@ def rekey_on_member(data, key, duplicates='error'):
 
     new_obj = {}
 
-    if isinstance(data, collections.Mapping):
+    if isinstance(data, Mapping):
         iterate_over = data.values()
-    elif isinstance(data, collections.Iterable) and not isinstance(data, (text_type, binary_type)):
+    elif isinstance(data, Iterable) and not isinstance(data, (text_type, binary_type)):
         iterate_over = data
     else:
         raise AnsibleFilterError("Type is not a valid list, set, or dict")
 
     for item in iterate_over:
-        if not isinstance(item, collections.Mapping):
+        if not isinstance(item, Mapping):
             raise AnsibleFilterError("List item is not a valid dict")
 
         try:

lib/ansible/plugins/filter/network.py

@@ -25,12 +25,12 @@ import os
 import traceback
 import string
 
-from collections import Mapping
 from xml.etree.ElementTree import fromstring
 
 from ansible.module_utils._text import to_text
 from ansible.module_utils.network.common.utils import Template
 from ansible.module_utils.six import iteritems, string_types
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.errors import AnsibleError, AnsibleFilterError
 from ansible.utils.encrypt import random_password
 

lib/ansible/plugins/inventory/__init__.py

@@ -24,12 +24,11 @@ import os
 import re
 import string
 
-from collections import Mapping
-
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.plugins import AnsiblePlugin
 from ansible.plugins.cache import InventoryFileCacheModule
 from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import string_types
 from ansible.template import Templar

lib/ansible/plugins/inventory/foreman.py

@@ -62,11 +62,11 @@ validate_certs: False
 
 import re
 
-from collections import MutableMapping
 from distutils.version import LooseVersion
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
 
 # 3rd party imports

lib/ansible/plugins/inventory/script.py

@@ -41,12 +41,12 @@ DOCUMENTATION = '''
 
 import os
 import subprocess
-from collections import Mapping
 
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.module_utils.basic import json_dict_bytes_to_unicode
 from ansible.module_utils.six import iteritems
 from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
 
 

lib/ansible/plugins/inventory/virtualbox.py

@@ -48,11 +48,11 @@ simple_config_file:
 
 import os
 
-from collections import MutableMapping
 from subprocess import Popen, PIPE
 
 from ansible.errors import AnsibleParserError
 from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
 
 

lib/ansible/plugins/inventory/yaml.py

@@ -60,11 +60,10 @@ all: # keys must be unique, i.e. only one 'hosts' per group
 
 import os
 
-from collections import MutableMapping
-
 from ansible.errors import AnsibleParserError
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.utils.addresses import parse_address
 from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range
 

lib/ansible/plugins/lookup/csvfile.py

@@ -60,12 +60,12 @@ RETURN = """
 
 import codecs
 import csv
-from collections import MutableSequence
 
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils.six import PY2
 from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.common._collections_compat import MutableSequence
 
 
 class CSVRecoder:

lib/ansible/plugins/lookup/dict.py

@@ -52,10 +52,10 @@ RETURN = """
       - list of composed dictonaries with key and value
     type: list
 """
-import collections
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
+from ansible.module_utils.common._collections_compat import Mapping
 
 
 class LookupModule(LookupBase):
@@ -69,7 +69,7 @@ class LookupModule(LookupBase):
         results = []
         for term in terms:
             # Expect any type of Mapping, notably hostvars
-            if not isinstance(term, collections.Mapping):
+            if not isinstance(term, Mapping):
                 raise AnsibleError("with_dict expects a dict")
 
             results.extend(self._flatten_hash_to_list(term))

lib/ansible/plugins/lookup/ini.py

@@ -62,12 +62,12 @@ _raw:
 """
 import os
 import re
-from collections import MutableSequence
 from io import StringIO
 
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.module_utils.six.moves import configparser
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.common._collections_compat import MutableSequence
 from ansible.plugins.lookup import LookupBase
 
 

lib/ansible/plugins/lookup/list.py

@@ -28,8 +28,8 @@ RETURN = """
   _list:
     description: basically the same as you fed in
 """
-import collections
 
+from ansible.module_utils.common._collections_compat import Sequence
 from ansible.plugins.lookup import LookupBase
 from ansible.errors import AnsibleError
 
@@ -37,6 +37,6 @@ from ansible.errors import AnsibleError
 class LookupModule(LookupBase):
 
     def run(self, terms, **kwargs):
-        if not isinstance(terms, collections.Sequence):
+        if not isinstance(terms, Sequence):
             raise AnsibleError("with_list expects a list")
         return terms

lib/ansible/plugins/test/core.py

@@ -21,10 +21,10 @@ __metaclass__ = type
 
 import re
 import operator as py_operator
-from collections import MutableMapping, MutableSequence
 from distutils.version import LooseVersion, StrictVersion
 
 from ansible import errors
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
 
 try:
     from __main__ import display