Python 3.8 collections compatibility fixes.

Includes a new pylint blacklist plugin to prevent regressions.

contrib/inventory/ssh_config.py

@@ -43,12 +43,12 @@
 import argparse
 import os.path
 import sys
-from collections import MutableSequence
 
 import json
 
 import paramiko
 
+from ansible.module_utils.common._collections_compat import MutableSequence
 
 SSH_CONF = '~/.ssh/config'
 

contrib/inventory/vmware.py

@@ -30,7 +30,6 @@ required if these options are specified using environment variables.
 
 from __future__ import print_function
 
-import collections
 import json
 import logging
 import optparse
@@ -42,6 +41,8 @@ import time
 from six import integer_types, text_type, string_types
 from six.moves import configparser
 
+from ansible.module_utils.common._collections_compat import MutableMapping
+
 # Disable logging message trigged by pSphere/suds.
 try:
     from logging import NullHandler
@@ -159,7 +160,7 @@ class VMwareInventory(object):
             if k.startswith('_'):
                 continue
             new_key = parent_key + sep + k if parent_key else k
-            if isinstance(v, collections.MutableMapping):
+            if isinstance(v, MutableMapping):
                 items.extend(self._flatten_dict(v, new_key, sep).items())
             elif isinstance(v, (list, tuple)):
                 if all([isinstance(x, string_types) for x in v]):
@@ -229,7 +230,7 @@ class VMwareInventory(object):
             except AttributeError:
                 host_info['%ss' % attr] = []
         for k, v in self._get_obj_info(host.summary, depth=0).items():
-            if isinstance(v, collections.MutableMapping):
+            if isinstance(v, MutableMapping):
                 for k2, v2 in v.items():
                     host_info[k2] = v2
             elif k != 'host':
@@ -265,7 +266,7 @@ class VMwareInventory(object):
         except AttributeError:
             vm_info['guestState'] = ''
         for k, v in self._get_obj_info(vm.summary, depth=0).items():
-            if isinstance(v, collections.MutableMapping):
+            if isinstance(v, MutableMapping):
                 for k2, v2 in v.items():
                     if k2 == 'host':
                         k2 = 'hostSystem'

lib/ansible/cli/doc.py

@@ -23,12 +23,11 @@ import textwrap
 import traceback
 import yaml
 
-from collections import Sequence
-
 from ansible import constants as C
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import Sequence
 from ansible.module_utils.six import string_types
 from ansible.parsing.metadata import extract_metadata
 from ansible.parsing.plugin_docs import read_docstub

lib/ansible/compat/selectors/_selectors2.py

@@ -24,7 +24,8 @@ import select
 import socket
 import sys
 import time
-from collections import namedtuple, Mapping
+from collections import namedtuple
+from ansible.module_utils.common._collections_compat import Mapping
 
 try:
     monotonic = time.monotonic

lib/ansible/errors/__init__.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import Sequence
 import traceback
 import sys
 
@@ -33,6 +32,7 @@ from ansible.errors.yaml_strings import (
     YAML_POSITION_DETAILS,
 )
 from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import Sequence
 
 
 class AnsibleError(Exception):

lib/ansible/executor/stats.py

@@ -19,8 +19,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import MutableMapping
-
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.utils.vars import merge_hash
 
 

lib/ansible/module_utils/common/_collections_compat.py

@@ -10,20 +10,37 @@ Third parties should not use this.
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-
 try:
     """Python 3.3+ branch."""
     from collections.abc import (
+        MappingView,
+        ItemsView,
         KeysView,
+        ValuesView,
         Mapping, MutableMapping,
         Sequence, MutableSequence,
         Set, MutableSet,
+        Container,
+        Hashable,
+        Sized,
+        Callable,
+        Iterable,
+        Iterator,
     )
 except ImportError:
     """Use old lib location under 2.6-3.2."""
     from collections import (
+        MappingView,
+        ItemsView,
         KeysView,
+        ValuesView,
         Mapping, MutableMapping,
         Sequence, MutableSequence,
         Set, MutableSet,
+        Container,
+        Hashable,
+        Sized,
+        Callable,
+        Iterable,
+        Iterator,
     )

lib/ansible/module_utils/network/nxos/nxos.py

@@ -36,6 +36,7 @@ from ansible.module_utils._text import to_text
 from ansible.module_utils.basic import env_fallback, return_values
 from ansible.module_utils.network.common.utils import to_list, ComplexList
 from ansible.module_utils.connection import Connection, ConnectionError
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.six import iteritems, string_types
 from ansible.module_utils.urls import fetch_url
@@ -179,7 +180,7 @@ class Cli:
         responses = []
         try:
             resp = connection.edit_config(config, replace=replace)
-            if isinstance(resp, collections.Mapping):
+            if isinstance(resp, Mapping):
                 resp = resp['response']
         except ConnectionError as e:
             code = getattr(e, 'code', 1)

lib/ansible/module_utils/oneview.py

@@ -43,6 +43,7 @@ except ImportError:
 from ansible.module_utils import six
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import Mapping
 
 
 def transform_list_to_dict(list_):
@@ -59,7 +60,7 @@ def transform_list_to_dict(list_):
         return ret
 
     for value in list_:
-        if isinstance(value, collections.Mapping):
+        if isinstance(value, Mapping):
             ret.update(value)
         else:
             ret[to_native(value, errors='surrogate_or_strict')] = True
@@ -108,7 +109,7 @@ def merge_list_by_key(original_list, updated_list, key, ignore_when_null=None):
 
 
 def _str_sorted(obj):
-    if isinstance(obj, collections.Mapping):
+    if isinstance(obj, Mapping):
         return json.dumps(obj, sort_keys=True)
     else:
         return str(obj)
@@ -430,7 +431,7 @@ class OneViewModuleBase(object):
             # If both values are null, empty or False it will be considered equal.
             elif not resource1[key] and not resource2[key]:
                 continue
-            elif isinstance(resource1[key], collections.Mapping):
+            elif isinstance(resource1[key], Mapping):
                 # recursive call
                 if not self.compare(resource1[key], resource2[key]):
                     self.module.log(self.MSG_DIFF_AT_KEY.format(key) + debug_resources)
@@ -482,7 +483,7 @@ class OneViewModuleBase(object):
         resource2 = sorted(resource2, key=_str_sorted)
 
         for i, val in enumerate(resource1):
-            if isinstance(val, collections.Mapping):
+            if isinstance(val, Mapping):
                 # change comparison function to compare dictionaries
                 if not self.compare(val, resource2[i]):
                     self.module.log("resources are different. " + debug_resources)

lib/ansible/module_utils/ovirt.py

@@ -18,7 +18,6 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import collections
 import inspect
 import os
 import time
@@ -27,6 +26,8 @@ from abc import ABCMeta, abstractmethod
 from datetime import datetime
 from distutils.version import LooseVersion
 
+from ansible.module_utils.common._collections_compat import Mapping
+
 try:
     from enum import Enum  # enum is a ovirtsdk4 requirement
     import ovirtsdk4 as sdk
@@ -512,7 +513,7 @@ class BaseModule(object):
 
     def diff_update(self, after, update):
         for k, v in update.items():
-            if isinstance(v, collections.Mapping):
+            if isinstance(v, Mapping):
                 after[k] = self.diff_update(after.get(k, dict()), v)
             else:
                 after[k] = update[k]

lib/ansible/modules/files/xml.py

@@ -266,7 +266,6 @@ import os
 import re
 import traceback
 
-from collections import MutableMapping
 from distutils.version import LooseVersion
 from io import BytesIO
 
@@ -279,6 +278,7 @@ except ImportError:
 from ansible.module_utils.basic import AnsibleModule, json_dict_bytes_to_unicode
 from ansible.module_utils.six import iteritems, string_types
 from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common._collections_compat import MutableMapping
 
 _IDENT = r"[a-zA-Z-][a-zA-Z0-9_\-\.]*"
 _NSIDENT = _IDENT + "|" + _IDENT + ":" + _IDENT

lib/ansible/modules/net_tools/basics/uri.py

@@ -270,11 +270,11 @@ import sys
 import tempfile
 import traceback
 
-from collections import Mapping, Sequence
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.six import PY2, iteritems, string_types
 from ansible.module_utils.six.moves.urllib.parse import urlencode, urlsplit
 from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import Mapping, Sequence
 from ansible.module_utils.urls import fetch_url, url_argument_spec
 
 JSON_CANDIDATES = ('text', 'json', 'javascript')

lib/ansible/parsing/ajson.py

@@ -7,10 +7,10 @@ __metaclass__ = type
 
 import json
 
-from collections import Mapping
 from datetime import date, datetime
 
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var
 from ansible.parsing.vault import VaultLib

lib/ansible/playbook/role/__init__.py

@@ -19,11 +19,11 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import os
 
 from ansible.errors import AnsibleError, AnsibleParserError, AnsibleAssertionError
 from ansible.module_utils.six import iteritems, binary_type, text_type
+from ansible.module_utils.common._collections_compat import Container, Mapping, Set, Sequence
 from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.become import Become
@@ -59,8 +59,8 @@ def hash_params(params):
     # Any container is unhashable if it contains unhashable items (for
     # instance, tuple() is a Hashable subclass but if it contains a dict, it
     # cannot be hashed)
-    if isinstance(params, collections.Container) and not isinstance(params, (text_type, binary_type)):
-        if isinstance(params, collections.Mapping):
+    if isinstance(params, Container) and not isinstance(params, (text_type, binary_type)):
+        if isinstance(params, Mapping):
             try:
                 # Optimistically hope the contents are all hashable
                 new_params = frozenset(params.items())
@@ -71,7 +71,7 @@ def hash_params(params):
                     new_params.update((k, hash_params(v)))
                 new_params = frozenset(new_params)
 
-        elif isinstance(params, (collections.Set, collections.Sequence)):
+        elif isinstance(params, (Set, Sequence)):
             try:
                 # Optimistically hope the contents are all hashable
                 new_params = frozenset(params)

lib/ansible/plugins/action/synchronize.py

@@ -18,11 +18,11 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os.path
-from collections import MutableSequence
 
 from ansible import constants as C
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import MutableSequence
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.action import ActionBase
 from ansible.plugins.loader import connection_loader

lib/ansible/plugins/cache/__init__.py

@@ -21,12 +21,12 @@ import os
 import time
 import errno
 from abc import ABCMeta, abstractmethod
-from collections import MutableMapping
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import with_metaclass
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.loader import cache_loader
 
 try:

lib/ansible/plugins/cache/memcached.py

@@ -50,6 +50,7 @@ from itertools import chain
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
+from ansible.module_utils.common._collections_compat import MutableSet
 from ansible.plugins.cache import BaseCacheModule
 
 try:
@@ -126,7 +127,7 @@ class ProxyClientPool(object):
             self.release_connection(conn)
 
 
-class CacheModuleKeys(collections.MutableSet):
+class CacheModuleKeys(MutableSet):
     """
     A set subclass that keeps track of insertion time and persists
     the set in memcached.

lib/ansible/plugins/callback/__init__.py

@@ -26,9 +26,9 @@ import sys
 import warnings
 
 from copy import deepcopy
-from collections import MutableMapping
 
 from ansible import constants as C
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins import AnsiblePlugin, get_plugin_class
 from ansible.utils.color import stringc

lib/ansible/plugins/callback/dense.py

@@ -20,8 +20,6 @@ requirements:
 - set as stdout in configuation
 '''
 
-from collections import MutableMapping, MutableSequence
-
 HAS_OD = False
 try:
     from collections import OrderedDict
@@ -30,6 +28,7 @@ except ImportError:
     pass
 
 from ansible.module_utils.six import binary_type, text_type
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
 from ansible.plugins.callback.default import CallbackModule as CallbackModule_default
 from ansible.utils.color import colorize, hostcolor
 

lib/ansible/plugins/callback/log_plays.py

@@ -21,9 +21,9 @@ DOCUMENTATION = '''
 import os
 import time
 import json
-from collections import MutableMapping
 
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.callback import CallbackBase
 
 

lib/ansible/plugins/cliconf/eos.py

@@ -41,12 +41,12 @@ options:
         version_added: '2.7'
 """
 
-import collections
 import json
 import time
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.utils import to_list
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.plugins.cliconf import CliconfBase, enable_mode
@@ -116,7 +116,7 @@ class Cliconf(CliconfBase):
         requests = []
         multiline = False
         for line in to_list(candidate):
-            if not isinstance(line, collections.Mapping):
+            if not isinstance(line, Mapping):
                 line = {'command': line}
 
             cmd = line['command']
@@ -175,7 +175,7 @@ class Cliconf(CliconfBase):
             raise ValueError("'commands' value is required")
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/ios.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import time
 import json
@@ -28,6 +27,7 @@ from itertools import chain
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
@@ -135,7 +135,7 @@ class Cliconf(CliconfBase):
         if commit:
             self.send_command('configure terminal')
             for line in to_list(candidate):
-                if not isinstance(line, collections.Mapping):
+                if not isinstance(line, Mapping):
                     line = {'command': line}
 
                 cmd = line['command']
@@ -279,7 +279,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/iosxr.py

@@ -19,12 +19,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import json
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.connection import ConnectionError
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
@@ -99,7 +99,7 @@ class Cliconf(CliconfBase):
             candidate = 'load {0}'.format(replace)
 
         for line in to_list(candidate):
-            if not isinstance(line, collections.Mapping):
+            if not isinstance(line, Mapping):
                 line = {'command': line}
             cmd = line['command']
             results.append(self.send_command(**line))
@@ -177,7 +177,7 @@ class Cliconf(CliconfBase):
             raise ValueError("'commands' value is required")
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/junos.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import json
 import re
 
@@ -27,6 +26,7 @@ from itertools import chain
 from functools import wraps
 
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase
 
@@ -100,7 +100,7 @@ class Cliconf(CliconfBase):
             candidate = 'load replace {0}'.format(replace)
 
         for line in to_list(candidate):
-            if not isinstance(line, collections.Mapping):
+            if not isinstance(line, Mapping):
                 line = {'command': line}
             cmd = line['command']
             results.append(self.send_command(**line))

lib/ansible/plugins/cliconf/nxos.py

@@ -19,12 +19,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import json
 import re
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.connection import ConnectionError
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
@@ -177,7 +177,7 @@ class Cliconf(CliconfBase):
             self.send_command('configure terminal')
 
             for line in to_list(candidate):
-                if not isinstance(line, collections.Mapping):
+                if not isinstance(line, Mapping):
                     line = {'command': line}
 
                 cmd = line['command']
@@ -204,7 +204,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/voss.py

@@ -19,12 +19,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import json
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase, enable_mode
@@ -126,7 +126,7 @@ class Cliconf(CliconfBase):
         if commit:
             self.send_command('configure terminal')
             for line in to_list(candidate):
-                if not isinstance(line, collections.Mapping):
+                if not isinstance(line, Mapping):
                     line = {'command': line}
 
                 cmd = line['command']
@@ -209,7 +209,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/cliconf/vyos.py

@@ -19,7 +19,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
 import re
 import json
 
@@ -27,6 +26,7 @@ from itertools import chain
 
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 from ansible.module_utils.network.common.utils import to_list
 from ansible.plugins.cliconf import CliconfBase
@@ -75,7 +75,7 @@ class Cliconf(CliconfBase):
         requests = []
         self.send_command('configure')
         for cmd in to_list(candidate):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             results.append(self.send_command(**cmd))
@@ -199,7 +199,7 @@ class Cliconf(CliconfBase):
 
         responses = list()
         for cmd in to_list(commands):
-            if not isinstance(cmd, collections.Mapping):
+            if not isinstance(cmd, Mapping):
                 cmd = {'command': cmd}
 
             output = cmd.pop('output', None)

lib/ansible/plugins/filter/core.py

@@ -34,7 +34,6 @@ import time
 import uuid
 import yaml
 
-from collections import MutableMapping
 import datetime
 from functools import partial
 from random import Random, SystemRandom, shuffle, random
@@ -46,6 +45,7 @@ from ansible.module_utils.six import iteritems, string_types, integer_types, rer
 from ansible.module_utils.six.moves import reduce, shlex_quote
 from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.common.collections import is_sequence
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.parsing.yaml.dumper import AnsibleDumper
 from ansible.utils.encrypt import passlib_or_crypt

lib/ansible/plugins/filter/mathstuff.py

@@ -23,7 +23,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 
-import collections
 import itertools
 import math
 
@@ -33,6 +32,7 @@ from ansible.errors import AnsibleFilterError
 from ansible.module_utils import basic
 from ansible.module_utils.six import binary_type, text_type
 from ansible.module_utils.six.moves import zip, zip_longest
+from ansible.module_utils.common._collections_compat import Hashable, Mapping, Iterable
 from ansible.module_utils._text import to_native, to_text
 
 try:
@@ -55,7 +55,7 @@ def unique(environment, a, case_sensitive=False, attribute=None):
     try:
         if HAS_UNIQUE:
             c = do_unique(environment, a, case_sensitive=case_sensitive, attribute=attribute)
-            if isinstance(a, collections.Hashable):
+            if isinstance(a, Hashable):
                 c = set(c)
             else:
                 c = list(c)
@@ -75,7 +75,7 @@ def unique(environment, a, case_sensitive=False, attribute=None):
             raise AnsibleFilterError("Ansible's unique filter does not support case_sensitive nor attribute parameters, "
                                      "you need a newer version of Jinja2 that provides their version of the filter.")
 
-        if isinstance(a, collections.Hashable):
+        if isinstance(a, Hashable):
             c = set(a)
         else:
             c = []
@@ -87,7 +87,7 @@ def unique(environment, a, case_sensitive=False, attribute=None):
 
 @environmentfilter
 def intersect(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) & set(b)
     else:
         c = unique(environment, [x for x in a if x in b])
@@ -96,7 +96,7 @@ def intersect(environment, a, b):
 
 @environmentfilter
 def difference(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) - set(b)
     else:
         c = unique(environment, [x for x in a if x not in b])
@@ -105,7 +105,7 @@ def difference(environment, a, b):
 
 @environmentfilter
 def symmetric_difference(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) ^ set(b)
     else:
         isect = intersect(environment, a, b)
@@ -115,7 +115,7 @@ def symmetric_difference(environment, a, b):
 
 @environmentfilter
 def union(environment, a, b):
-    if isinstance(a, collections.Hashable) and isinstance(b, collections.Hashable):
+    if isinstance(a, Hashable) and isinstance(b, Hashable):
         c = set(a) | set(b)
     else:
         c = unique(environment, a + b)
@@ -189,15 +189,15 @@ def rekey_on_member(data, key, duplicates='error'):
 
     new_obj = {}
 
-    if isinstance(data, collections.Mapping):
+    if isinstance(data, Mapping):
         iterate_over = data.values()
-    elif isinstance(data, collections.Iterable) and not isinstance(data, (text_type, binary_type)):
+    elif isinstance(data, Iterable) and not isinstance(data, (text_type, binary_type)):
         iterate_over = data
     else:
         raise AnsibleFilterError("Type is not a valid list, set, or dict")
 
     for item in iterate_over:
-        if not isinstance(item, collections.Mapping):
+        if not isinstance(item, Mapping):
             raise AnsibleFilterError("List item is not a valid dict")
 
         try:

lib/ansible/plugins/filter/network.py

@@ -25,12 +25,12 @@ import os
 import traceback
 import string
 
-from collections import Mapping
 from xml.etree.ElementTree import fromstring
 
 from ansible.module_utils._text import to_text
 from ansible.module_utils.network.common.utils import Template
 from ansible.module_utils.six import iteritems, string_types
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.errors import AnsibleError, AnsibleFilterError
 from ansible.utils.encrypt import random_password
 

lib/ansible/plugins/inventory/__init__.py

@@ -24,12 +24,11 @@ import os
 import re
 import string
 
-from collections import Mapping
-
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.plugins import AnsiblePlugin
 from ansible.plugins.cache import InventoryFileCacheModule
 from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import string_types
 from ansible.template import Templar

lib/ansible/plugins/inventory/foreman.py

@@ -62,11 +62,11 @@ validate_certs: False
 
 import re
 
-from collections import MutableMapping
 from distutils.version import LooseVersion
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
 
 # 3rd party imports

lib/ansible/plugins/inventory/script.py

@@ -41,12 +41,12 @@ DOCUMENTATION = '''
 
 import os
 import subprocess
-from collections import Mapping
 
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.module_utils.basic import json_dict_bytes_to_unicode
 from ansible.module_utils.six import iteritems
 from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
 
 

lib/ansible/plugins/inventory/virtualbox.py

@@ -48,11 +48,11 @@ simple_config_file:
 
 import os
 
-from collections import MutableMapping
 from subprocess import Popen, PIPE
 
 from ansible.errors import AnsibleParserError
 from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
 
 

lib/ansible/plugins/inventory/yaml.py

@@ -60,11 +60,10 @@ all: # keys must be unique, i.e. only one 'hosts' per group
 
 import os
 
-from collections import MutableMapping
-
 from ansible.errors import AnsibleParserError
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.utils.addresses import parse_address
 from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range
 

lib/ansible/plugins/lookup/csvfile.py

@@ -60,12 +60,12 @@ RETURN = """
 
 import codecs
 import csv
-from collections import MutableSequence
 
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils.six import PY2
 from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.common._collections_compat import MutableSequence
 
 
 class CSVRecoder:

lib/ansible/plugins/lookup/dict.py

@@ -52,10 +52,10 @@ RETURN = """
       - list of composed dictonaries with key and value
     type: list
 """
-import collections
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
+from ansible.module_utils.common._collections_compat import Mapping
 
 
 class LookupModule(LookupBase):
@@ -69,7 +69,7 @@ class LookupModule(LookupBase):
         results = []
         for term in terms:
             # Expect any type of Mapping, notably hostvars
-            if not isinstance(term, collections.Mapping):
+            if not isinstance(term, Mapping):
                 raise AnsibleError("with_dict expects a dict")
 
             results.extend(self._flatten_hash_to_list(term))

lib/ansible/plugins/lookup/ini.py

@@ -62,12 +62,12 @@ _raw:
 """
 import os
 import re
-from collections import MutableSequence
 from io import StringIO
 
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.module_utils.six.moves import configparser
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.common._collections_compat import MutableSequence
 from ansible.plugins.lookup import LookupBase
 
 

lib/ansible/plugins/lookup/list.py

@@ -28,8 +28,8 @@ RETURN = """
   _list:
     description: basically the same as you fed in
 """
-import collections
 
+from ansible.module_utils.common._collections_compat import Sequence
 from ansible.plugins.lookup import LookupBase
 from ansible.errors import AnsibleError
 
@@ -37,6 +37,6 @@ from ansible.errors import AnsibleError
 class LookupModule(LookupBase):
 
     def run(self, terms, **kwargs):
-        if not isinstance(terms, collections.Sequence):
+        if not isinstance(terms, Sequence):
             raise AnsibleError("with_list expects a list")
         return terms

lib/ansible/plugins/test/core.py

@@ -21,10 +21,10 @@ __metaclass__ = type
 
 import re
 import operator as py_operator
-from collections import MutableMapping, MutableSequence
 from distutils.version import LooseVersion, StrictVersion
 
 from ansible import errors
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
 
 try:
     from __main__ import display

lib/ansible/template/__init__.py

@@ -27,7 +27,6 @@ import pwd
 import re
 import time
 
-from collections import Sequence, Mapping
 from functools import wraps
 from io import StringIO
 from numbers import Number
@@ -45,6 +44,7 @@ from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable, AnsibleAssertionError
 from ansible.module_utils.six import string_types, text_type
 from ansible.module_utils._text import to_native, to_text, to_bytes
+from ansible.module_utils.common._collections_compat import Sequence, Mapping
 from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
 from ansible.template.safe_eval import safe_eval
 from ansible.template.template import AnsibleJ2Template

lib/ansible/template/vars.py

@@ -19,13 +19,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import Mapping
-
 from jinja2.utils import missing
 
 from ansible.errors import AnsibleError, AnsibleUndefinedVariable
 from ansible.module_utils.six import iteritems
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import Mapping
 
 
 __all__ = ['AnsibleJ2Vars']

lib/ansible/utils/listify.py

@@ -19,9 +19,8 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import Iterable
-
 from ansible.module_utils.six import string_types
+from ansible.module_utils.common._collections_compat import Iterable
 from ansible.template.safe_eval import safe_eval
 
 

lib/ansible/utils/plugin_docs.py

@@ -4,11 +4,10 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import MutableMapping, MutableSet, MutableSequence
-
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSet, MutableSequence
 from ansible.parsing.plugin_docs import read_docstring, read_docstub
 from ansible.parsing.yaml.loader import AnsibleLoader
 

lib/ansible/utils/py3compat.py

@@ -13,10 +13,10 @@ __metaclass__ = type
 
 import os
 import sys
-from collections import MutableMapping
 
 from ansible.module_utils.six import PY3
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.common._collections_compat import MutableMapping
 
 __all__ = ('environ',)
 

lib/ansible/utils/unsafe_proxy.py

@@ -53,10 +53,9 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import Mapping, MutableSequence, Set
-
 from ansible.module_utils.six import string_types, text_type
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping, MutableSequence, Set
 
 
 __all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'wrap_var']

lib/ansible/utils/vars.py

@@ -23,7 +23,6 @@ import ast
 import random
 import uuid
 
-from collections import MutableMapping
 from json import dumps
 
 
@@ -31,6 +30,7 @@ from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.module_utils.six import iteritems, string_types
 from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.parsing.splitter import parse_kv
 
 

lib/ansible/vars/hostvars.py

@@ -19,11 +19,10 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
-
 from jinja2.runtime import Undefined
 
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import Mapping
 from ansible.template import Templar
 
 STATIC_VARS = [
@@ -51,7 +50,7 @@ __all__ = ['HostVars', 'HostVarsVars']
 
 
 # Note -- this is a Mapping, not a MutableMapping
-class HostVars(collections.Mapping):
+class HostVars(Mapping):
     ''' A special view of vars_cache that adds values from the inventory when needed. '''
 
     def __init__(self, inventory, variable_manager, loader):
@@ -117,7 +116,7 @@ class HostVars(collections.Mapping):
         return repr(out)
 
 
-class HostVarsVars(collections.Mapping):
+class HostVarsVars(Mapping):
 
     def __init__(self, variables, loader):
         self._vars = variables

lib/ansible/vars/manager.py

@@ -22,7 +22,7 @@ __metaclass__ = type
 import os
 import sys
 
-from collections import defaultdict, MutableMapping, Sequence
+from collections import defaultdict
 
 try:
     from hashlib import sha1
@@ -36,6 +36,7 @@ from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVar
 from ansible.inventory.host import Host
 from ansible.inventory.helpers import sort_groups, get_group_vars
 from ansible.module_utils._text import to_native
+from ansible.module_utils.common._collections_compat import MutableMapping, Sequence
 from ansible.module_utils.six import iteritems, text_type, string_types
 from ansible.plugins.loader import lookup_loader, vars_loader
 from ansible.plugins.cache import FactCache

test/sanity/pylint/ignore.txt

@@ -63,3 +63,4 @@ lib/ansible/modules/storage/infinidat/infini_vol.py ansible-format-automatic-spe
 lib/ansible/modules/storage/purestorage/purefa_host.py ansible-format-automatic-specification
 lib/ansible/modules/storage/purestorage/purefa_pg.py ansible-format-automatic-specification
 lib/ansible/modules/system/firewalld.py ansible-format-automatic-specification
+test/units/modules/system/test_known_hosts.py ansible-bad-function

test/sanity/pylint/plugins/blacklist.py

@@ -0,0 +1,200 @@
+"""A plugin for pylint to identify imports and functions which should not be used."""
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+import astroid
+
+from pylint.checkers import BaseChecker
+from pylint.interfaces import IAstroidChecker
+
+
+class BlacklistEntry(object):
+    """Defines a import blacklist entry."""
+    def __init__(self, alternative, modules_only=False, names=None, ignore_paths=None):
+        """
+        :type alternative: str
+        :type modules_only: bool
+        :type names: tuple[str] | None
+        :type ignore_paths: tuple[str] | None
+        """
+        self.alternative = alternative
+        self.modules_only = modules_only
+        self.names = set(names) if names else None
+        self.ignore_paths = ignore_paths
+
+    def applies_to(self, path, name=None):
+        """
+        :type path: str
+        :type name: str | None
+        :rtype: bool
+        """
+        if self.names:
+            if not name:
+                return False
+
+            if name not in self.names:
+                return False
+
+        if self.ignore_paths and any(path.endswith(ignore_path) for ignore_path in self.ignore_paths):
+            return False
+
+        if self.modules_only:
+            return '/lib/ansible/modules/' in path or '/lib/ansible/module_utils/' in path
+
+        return True
+
+
+class AnsibleBlacklistChecker(BaseChecker):
+    """Checker for blacklisted imports and functions."""
+    __implements__ = (IAstroidChecker,)
+
+    name = 'blacklist'
+
+    BAD_IMPORT = 'ansible-bad-import'
+    BAD_IMPORT_FROM = 'ansible-bad-import-from'
+    BAD_FUNCTION = 'ansible-bad-function'
+
+    msgs = dict(
+        E5101=('Import %s instead of %s',
+               BAD_IMPORT,
+               'Identifies imports which should not be used.'),
+        E5102=('Import %s from %s instead of %s',
+               BAD_IMPORT_FROM,
+               'Identifies imports which should not be used.'),
+        E5103=('Call %s instead of %s',
+               BAD_FUNCTION,
+               'Identifies functions which should not be used.'),
+    )
+
+    blacklist_imports = dict(
+        # Additional imports that we may want to start checking:
+        # boto=BlacklistEntry('boto3', modules_only=True),
+        # requests=BlacklistEntry('ansible.module_utils.urls', modules_only=True),
+        # urllib=BlacklistEntry('ansible.module_utils.urls', modules_only=True),
+
+        # see https://docs.python.org/2/library/urllib2.html
+        urllib2=BlacklistEntry('ansible.module_utils.urls',
+                               ignore_paths=(
+                                   '/lib/ansible/module_utils/urls.py',
+                               )),
+
+        # see https://docs.python.org/3.7/library/collections.abc.html
+        collections=BlacklistEntry('ansible.module_utils.common._collections_compat',
+                                   ignore_paths=(
+                                       '/lib/ansible/module_utils/common/_collections_compat.py',
+                                   ),
+                                   names=(
+                                       'MappingView',
+                                       'ItemsView',
+                                       'KeysView',
+                                       'ValuesView',
+                                       'Mapping', 'MutableMapping',
+                                       'Sequence', 'MutableSequence',
+                                       'Set', 'MutableSet',
+                                       'Container',
+                                       'Hashable',
+                                       'Sized',
+                                       'Callable',
+                                       'Iterable',
+                                       'Iterator',
+                                   )),
+    )
+
+    blacklist_functions = {
+        # see https://docs.python.org/2/library/tempfile.html#tempfile.mktemp
+        'tempfile.mktemp': BlacklistEntry('tempfile.mkstemp'),
+
+        'sys.exit': BlacklistEntry('exit_json or fail_json',
+                                   ignore_paths=(
+                                       '/lib/ansible/module_utils/basic.py',
+                                       '/lib/ansible/modules/utilities/logic/async_wrapper.py',
+                                       '/lib/ansible/module_utils/common/removed.py',
+                                   ),
+                                   modules_only=True),
+    }
+
+    def visit_import(self, node):
+        """
+        :type node: astroid.node_classes.Import
+        """
+        for name in node.names:
+            self._check_import(node, name[0])
+
+    def visit_importfrom(self, node):
+        """
+        :type node: astroid.node_classes.ImportFrom
+        """
+        self._check_importfrom(node, node.modname, node.names)
+
+    def visit_attribute(self, node):
+        """
+        :type node: astroid.node_classes.Attribute
+        """
+        last_child = node.last_child()
+
+        # this is faster than using type inference and will catch the most common cases
+        if not isinstance(last_child, astroid.node_classes.Name):
+            return
+
+        module = last_child.name
+
+        entry = self.blacklist_imports.get(module)
+
+        if entry and entry.names:
+            if entry.applies_to(self.linter.current_file, node.attrname):
+                self.add_message(self.BAD_IMPORT_FROM, args=(node.attrname, entry.alternative, module), node=node)
+
+    def visit_call(self, node):
+        """
+        :type node: astroid.node_classes.Call
+        """
+        try:
+            for i in node.func.inferred():
+                func = None
+
+                if isinstance(i, astroid.scoped_nodes.FunctionDef) and isinstance(i.parent, astroid.scoped_nodes.Module):
+                    func = '%s.%s' % (i.parent.name, i.name)
+
+                if not func:
+                    continue
+
+                entry = self.blacklist_functions.get(func)
+
+                if entry and entry.applies_to(self.linter.current_file):
+                    self.add_message(self.BAD_FUNCTION, args=(entry.alternative, func), node=node)
+        except astroid.exceptions.InferenceError:
+            pass
+
+    def _check_import(self, node, modname):
+        """
+        :type node: astroid.node_classes.Import
+        :type modname: str
+        """
+        entry = self.blacklist_imports.get(modname)
+
+        if not entry:
+            return
+
+        if entry.applies_to(self.linter.current_file):
+            self.add_message(self.BAD_IMPORT, args=(entry.alternative, modname), node=node)
+
+    def _check_importfrom(self, node, modname, names):
+        """
+        :type node: astroid.node_classes.ImportFrom
+        :type modname: str
+        :type names:  list[str[
+        """
+        entry = self.blacklist_imports.get(modname)
+
+        if not entry:
+            return
+
+        for name in names:
+            if entry.applies_to(self.linter.current_file, name[0]):
+                self.add_message(self.BAD_IMPORT_FROM, args=(name[0], entry.alternative, modname), node=node)
+
+
+def register(linter):
+    """required method to auto register this checker """
+    linter.register_checker(AnsibleBlacklistChecker(linter))

test/units/module_utils/conftest.py

@@ -3,7 +3,6 @@
 
 import json
 import sys
-from collections import MutableMapping
 from io import BytesIO
 
 import pytest
@@ -11,6 +10,7 @@ import pytest
 import ansible.module_utils.basic
 from ansible.module_utils.six import PY3, string_types
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 
 
 @pytest.fixture

test/units/modules/conftest.py

@@ -2,12 +2,12 @@
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 import json
-from collections import MutableMapping
 
 import pytest
 
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common._collections_compat import MutableMapping
 
 
 @pytest.fixture

test/units/modules/web_infrastructure/test_jenkins_plugin.py

@@ -1,7 +1,7 @@
-import collections
 from io import BytesIO
 
 from ansible.modules.web_infrastructure.jenkins_plugin import JenkinsPlugin
+from ansible.module_utils.common._collections_compat import Mapping
 
 
 def pass_function(*args, **kwargs):
@@ -145,4 +145,4 @@ def test__get_json_data(mocker):
         "{url}".format(url=GITHUB_DATA['url']),
         'CSRF')
 
-    assert isinstance(json_data, collections.Mapping)
+    assert isinstance(json_data, Mapping)

test/units/parsing/yaml/test_loader.py

@@ -22,12 +22,11 @@ __metaclass__ = type
 
 from io import StringIO
 
-from collections import Sequence, Set, Mapping
-
 from ansible.compat.tests import unittest
 
 from ansible import errors
 from ansible.module_utils.six import text_type, binary_type
+from ansible.module_utils.common._collections_compat import Sequence, Set, Mapping
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing import vault
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode

test/units/playbook/role/test_role.py

@@ -19,12 +19,11 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import collections
-
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock
 
 from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.module_utils.common._collections_compat import Container
 from ansible.playbook.block import Block
 from ansible.playbook.task import Task
 
@@ -97,7 +96,7 @@ class TestHashParams(unittest.TestCase):
 
     def test_container_but_not_iterable(self):
         # This is a Container that is not iterable, which is unlikely but...
-        class MyContainer(collections.Container):
+        class MyContainer(Container):
             def __init__(self, some_thing):
                 self.data = []
                 self.data.append(some_thing)