Combine jimi-c and bcoca's ideas and work on hooking module-utils into PluginLoader.

This version just gets the relevant paths from PluginLoader and then
uses the existing imp.find_plugin() calls in the AnsiballZ code to load
the proper module_utils.

Modify PluginLoader to optionally omit subdirectories (module_utils
needs to operate on top level dirs, not on subdirs because it has
a hierarchical namespace whereas all other plugins use a flat
namespace).

Rename snippet* variables to module_utils*

Add a small number of unittests for recursive_finder

Add a larger number of integration tests to demonstrate that
module_utils is working.

Whitelist module-style shebang in test target library dirs

Prefix module_data variable with b_ to be clear that it holds bytes data

lib/ansible/executor/module_common.py

@@ -34,6 +34,7 @@ from ansible.release import __version__, __author__
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.plugins import module_utils_loader
 # Must import strategy and use write_locks from there
 # If we import write_locks directly then we end up binding a
 # variable to the object and then it never gets updated.
@@ -57,8 +58,8 @@ REPLACER_SELINUX  = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"
 # specify an encoding for the python source file
 ENCODING_STRING = u'# -*- coding: utf-8 -*-'
 
-# we've moved the module_common relative to the snippets, so fix the path
+# module_common is relative to module_utils, so fix the path
-_SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
+_MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
 
 # ******************************************************************************
 
@@ -468,13 +469,16 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
     # Loop through the imports that we've found to normalize them
     # Exclude paths that match with paths we've already processed
     # (Have to exclude them a second time once the paths are processed)
+
+    module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)]
+    module_utils_paths.append(_MODULE_UTILS_PATH)
     for py_module_name in finder.submodules.difference(py_module_names):
         module_info = None
 
         if py_module_name[0] == 'six':
             # Special case the python six library because it messes up the
             # import process in an incompatible way
-            module_info = imp.find_module('six', [_SNIPPET_PATH])
+            module_info = imp.find_module('six', module_utils_paths)
             py_module_name = ('six',)
             idx = 0
         else:
@@ -485,7 +489,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
                     break
                 try:
                     module_info = imp.find_module(py_module_name[-idx],
-                            [os.path.join(_SNIPPET_PATH, *py_module_name[:-idx])])
+                            [os.path.join(p, *py_module_name[:-idx]) for p in module_utils_paths])
                     break
                 except ImportError:
                     continue
@@ -513,7 +517,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
                 if module_info[2][2] == imp.PKG_DIRECTORY:
                     # Read the __init__.py instead of the module file as this is
                     # a python package
-                    py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(_SNIPPET_PATH, *py_module_name), '__init__.py'))
+                    py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(module_info[1], '__init__.py')))
                     normalized_modules.add(py_module_name + ('__init__',))
                 else:
                     py_module_cache[py_module_name] = module_info[0].read()
@@ -525,8 +529,10 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
             for i in range(1, len(py_module_name)):
                 py_pkg_name = py_module_name[:-i] + ('__init__',)
                 if py_pkg_name not in py_module_names:
+                    pkg_dir_info = imp.find_module(py_pkg_name[-1],
+                            [os.path.join(p, *py_pkg_name[:-1]) for p in module_utils_paths])
                     normalized_modules.add(py_pkg_name)
-                    py_module_cache[py_pkg_name] = _slurp('%s.py' % os.path.join(_SNIPPET_PATH, *py_pkg_name))
+                    py_module_cache[py_pkg_name] = _slurp(pkg_dir_info[1])
 
     #
     # iterate through all of the ansible.module_utils* imports that we haven't
@@ -554,13 +560,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
         del py_module_cache[py_module_file]
 
 
-def _is_binary(module_data):
+def _is_binary(b_module_data):
     textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
-    start = module_data[:1024]
+    start = b_module_data[:1024]
     return bool(start.translate(None, textchars))
 
 
-def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
+def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression):
     """
     Given the source of the module, convert it to a Jinja2 template to insert
     module code and return whether it's a new or old style module.
@@ -574,31 +580,31 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
     # module_substyle is extra information that's useful internally.  It tells
     # us what we have to look to substitute in the module files and whether
     # we're using module replacer or ansiballz to format the module itself.
-    if _is_binary(module_data):
+    if _is_binary(b_module_data):
         module_substyle = module_style = 'binary'
-    elif REPLACER in module_data:
+    elif REPLACER in b_module_data:
         # Do REPLACER before from ansible.module_utils because we need make sure
         # we substitute "from ansible.module_utils basic" for REPLACER
         module_style = 'new'
         module_substyle = 'python'
-        module_data = module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
+        b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
-    elif b'from ansible.module_utils.' in module_data:
+    elif b'from ansible.module_utils.' in b_module_data:
         module_style = 'new'
         module_substyle = 'python'
-    elif REPLACER_WINDOWS in module_data:
+    elif REPLACER_WINDOWS in b_module_data:
         module_style = 'new'
         module_substyle = 'powershell'
-    elif REPLACER_JSONARGS in module_data:
+    elif REPLACER_JSONARGS in b_module_data:
         module_style = 'new'
         module_substyle = 'jsonargs'
-    elif b'WANT_JSON' in module_data:
+    elif b'WANT_JSON' in b_module_data:
         module_substyle = module_style = 'non_native_want_json'
 
     shebang = None
     # Neither old-style, non_native_want_json nor binary modules should be modified
     # except for the shebang line (Done by modify_module)
     if module_style in ('old', 'non_native_want_json', 'binary'):
-        return module_data, module_style, shebang
+        return b_module_data, module_style, shebang
 
     output = BytesIO()
     py_module_names = set()
@@ -651,10 +657,10 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
                             to_bytes(__author__) + b'"\n')
                     zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n')
 
-                    zf.writestr('ansible_module_%s.py' % module_name, module_data)
+                    zf.writestr('ansible_module_%s.py' % module_name, b_module_data)
 
                     py_module_cache = { ('__init__',): b'' }
-                    recursive_finder(module_name, module_data, py_module_names, py_module_cache, zf)
+                    recursive_finder(module_name, b_module_data, py_module_names, py_module_cache, zf)
                     zf.close()
                     zipdata = base64.b64encode(zipoutput.getvalue())
 
@@ -712,22 +718,23 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
             minute=now.minute,
             second=now.second,
         )))
-        module_data = output.getvalue()
+        b_module_data = output.getvalue()
 
     elif module_substyle == 'powershell':
         # Module replacer for jsonargs and windows
-        lines = module_data.split(b'\n')
+        lines = b_module_data.split(b'\n')
         for line in lines:
             if REPLACER_WINDOWS in line:
-                ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
+                # FIXME: Need to make a module_utils loader for powershell at some point
+                ps_data = _slurp(os.path.join(_MODULE_UTILS_PATH, "powershell.ps1"))
                 output.write(ps_data)
                 py_module_names.add((b'powershell',))
                 continue
             output.write(line + b'\n')
-        module_data = output.getvalue()
+        b_module_data = output.getvalue()
 
         module_args_json = to_bytes(json.dumps(module_args))
-        module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
+        b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)
 
         # Powershell/winrm don't actually make use of shebang so we can
         # safely set this here.  If we let the fallback code handle this
@@ -751,17 +758,17 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
         # ansiballz) If we remove them from jsonargs-style module replacer
         # then we can remove them everywhere.
         python_repred_args = to_bytes(repr(module_args_json))
-        module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
+        b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
-        module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
+        b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args)
-        module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
+        b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
 
         # The main event -- substitute the JSON args string into the module
-        module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
+        b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)
 
         facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict')
-        module_data = module_data.replace(b'syslog.LOG_USER', facility)
+        b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility)
 
-    return (module_data, module_style, shebang)
+    return (b_module_data, module_style, shebang)
 
 
 def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
@@ -791,14 +798,14 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
     with open(module_path, 'rb') as f:
 
         # read in the module source
-        module_data = f.read()
+        b_module_data = f.read()
 
-    (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
+    (b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression)
 
     if module_style == 'binary':
-        return (module_data, module_style, to_text(shebang, nonstring='passthru'))
+        return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))
     elif shebang is None:
-        lines = module_data.split(b"\n", 1)
+        lines = b_module_data.split(b"\n", 1)
         if lines[0].startswith(b"#!"):
             shebang = lines[0].strip()
             args = shlex.split(str(shebang[2:]))
@@ -815,8 +822,8 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
             # No shebang, assume a binary module?
             pass
 
-        module_data = b"\n".join(lines)
+        b_module_data = b"\n".join(lines)
     else:
         shebang = to_bytes(shebang, errors='surrogate_or_strict')
 
-    return (module_data, module_style, to_text(shebang, nonstring='passthru'))
+    return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))

lib/ansible/plugins/__init__.py

@@ -1,5 +1,6 @@
 # (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
 # (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> and others
+# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com>
 #
 # This file is part of Ansible
 #
@@ -31,6 +32,7 @@ import warnings
 from collections import defaultdict
 
 from ansible import constants as C
+from ansible.compat.six import string_types
 from ansible.module_utils._text import to_text
 
 
@@ -148,7 +150,7 @@ class PluginLoader:
                     results.append(os.path.join(root,x))
         return results
 
-    def _get_package_paths(self):
+    def _get_package_paths(self, subdirs=True):
         ''' Gets the path of a Python package '''
 
         if not self.package:
@@ -159,11 +161,18 @@ class PluginLoader:
             for parent_mod in parts:
                 m = getattr(m, parent_mod)
             self.package_path = os.path.dirname(m.__file__)
+        if subdirs:
             return self._all_directories(self.package_path)
+        return [self.package_path]
 
-    def _get_paths(self):
+    def _get_paths(self, subdirs=True):
         ''' Return a list of paths to search for plugins in '''
 
+        # FIXME: This is potentially buggy if subdirs is sometimes True and
+        # sometimes False.  In current usage, everything calls this with
+        # subdirs=True except for module_utils_loader which always calls it
+        # with subdirs=False.  So there currently isn't a problem with this
+        # caching.
         if self._paths is not None:
             return self._paths
 
@@ -173,6 +182,7 @@ class PluginLoader:
         if self.config is not None:
             for path in self.config:
                 path = os.path.realpath(os.path.expanduser(path))
+                if subdirs:
                     contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path)
                     for c in contents:
                         if os.path.isdir(c) and c not in ret:
@@ -181,7 +191,9 @@ class PluginLoader:
                     ret.append(path)
 
         # look for any plugins installed in the package subtree
-        ret.extend(self._get_package_paths())
+        # Note package path always gets added last so that every other type of
+        # path is searched before it.
+        ret.extend(self._get_package_paths(subdirs=subdirs))
 
         # HACK: because powershell modules are in the same directory
         # hierarchy as other modules we have to process them last.  This is
@@ -197,6 +209,7 @@ class PluginLoader:
         # would have class_names, they would not work as written.
         reordered_paths = []
         win_dirs = []
+
         for path in ret:
             if path.endswith('windows'):
                 win_dirs.append(path)
@@ -260,6 +273,9 @@ class PluginLoader:
 
                 # HACK: We have no way of executing python byte
                 # compiled files as ansible modules so specifically exclude them
+                ### FIXME: I believe this is only correct for modules and
+                # module_utils.  For all other plugins we want .pyc and .pyo should
+                # bew valid
                 if full_path.endswith(('.pyc', '.pyo')):
                     continue
 
@@ -466,6 +482,13 @@ module_loader = PluginLoader(
     'library',
 )
 
+module_utils_loader = PluginLoader(
+    '',
+    'ansible.module_utils',
+    'module_utils',
+    'module_utils',
+)
+
 lookup_loader = PluginLoader(
     'LookupModule',
     'ansible.plugins.lookup',

test/integration/targets/module_utils/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/module_utils/library/test.py

@@ -0,0 +1,83 @@
+#!/usr/bin/python
+
+results = {}
+
+# Test import with no from
+import ansible.module_utils.foo0
+results['foo0'] = ansible.module_utils.foo0.data
+
+# Test depthful import with no from
+import ansible.module_utils.bar0.foo
+results['bar0'] = ansible.module_utils.bar0.foo.data
+
+# Test import of module_utils/foo1.py
+from ansible.module_utils import foo1
+results['foo1'] = foo1.data
+
+# Test import of an identifier inside of module_utils/foo2.py
+from ansible.module_utils.foo2 import data
+results['foo2'] = data
+
+# Test import of module_utils/bar1/__init__.py
+from ansible.module_utils import bar1
+results['bar1'] = bar1.data
+
+# Test import of an identifier inside of module_utils/bar2/__init__.py
+from ansible.module_utils.bar2 import data
+results['bar2'] = data
+
+# Test import of module_utils/baz1/one.py
+from ansible.module_utils.baz1 import one
+results['baz1'] = one.data
+
+# Test import of an identifier inside of module_utils/baz2/one.py
+from ansible.module_utils.baz2.one import data
+results['baz2'] = data
+
+# Test import of module_utils/spam1/ham/eggs/__init__.py
+from ansible.module_utils.spam1.ham import eggs
+results['spam1'] = eggs.data
+
+# Test import of an identifier inside module_utils/spam2/ham/eggs/__init__.py
+from ansible.module_utils.spam2.ham.eggs import data
+results['spam2'] = data
+
+# Test import of module_utils/spam3/ham/bacon.py
+from ansible.module_utils.spam3.ham import bacon
+results['spam3'] = bacon.data
+
+# Test import of an identifier inside of module_utils/spam4/ham/bacon.py
+from ansible.module_utils.spam4.ham.bacon import data
+results['spam4'] = data
+
+# Test import of module_utils.spam5.ham bacon and eggs (modules)
+from ansible.module_utils.spam5.ham import bacon, eggs
+results['spam5'] = (bacon.data, eggs.data)
+
+# Test import of module_utils.spam6.ham bacon and eggs (identifiers)
+from ansible.module_utils.spam6.ham import bacon, eggs
+results['spam6'] = (bacon, eggs)
+
+# Test import of module_utils.spam7.ham bacon and eggs (module and identifier)
+from ansible.module_utils.spam7.ham import bacon, eggs
+results['spam7'] = (bacon.data, eggs)
+
+# Test import of module_utils/spam8/ham/bacon.py and module_utils/spam8/ham/eggs.py separately
+from ansible.module_utils.spam8.ham import bacon
+from ansible.module_utils.spam8.ham import eggs
+results['spam8'] = (bacon.data, eggs)
+
+# Test that import of module_utils/qux1/quux.py using as works
+from ansible.module_utils.qux1 import quux as one
+results['qux1'] = one.data
+
+# Test that importing qux2/quux.py and qux2/quuz.py using as works
+from ansible.module_utils.qux2 import quux as one, quuz as two
+results['qux2'] = (one.data, two.data)
+
+# Test depth
+from ansible.module_utils.a.b.c.d.e.f.g.h import data
+
+results['abcdefgh'] = data
+from ansible.module_utils.basic import AnsibleModule
+AnsibleModule(argument_spec=dict()).exit_json(**results)

test/integration/targets/module_utils/library/test_failure.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+
+results = {}
+# Test that we are rooted correctly
+# Following files:
+#   module_utils/yak/zebra/foo.py
+try:
+    from ansible.module_utils.zebra import foo
+    results['zebra'] = foo.data
+except ImportError:
+    results['zebra'] = 'Failed in module as expected but incorrectly did not fail in AnsiballZ construction'
+
+from ansible.module_utils.basic import AnsibleModule
+AnsibleModule(argument_spec=dict()).exit_json(**results)

test/integration/targets/module_utils/library/test_override.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.facts import data
+
+results = {"data": data}
+
+AnsibleModule(argument_spec=dict()).exit_json(**results)

test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/g/h/__init__.py

@@ -0,0 +1 @@
+data = 'abcdefgh'

test/integration/targets/module_utils/module_utils/bar0/foo.py

@@ -0,0 +1 @@
+data = 'bar0'

test/integration/targets/module_utils/module_utils/bar1/__init__.py

@@ -0,0 +1 @@
+data = 'bar1'

test/integration/targets/module_utils/module_utils/bar2/__init__.py

@@ -0,0 +1 @@
+data = 'bar2'

test/integration/targets/module_utils/module_utils/baz1/one.py

@@ -0,0 +1 @@
+data = 'baz1'

test/integration/targets/module_utils/module_utils/baz2/one.py

@@ -0,0 +1 @@
+data = 'baz2'

test/integration/targets/module_utils/module_utils/facts.py

@@ -0,0 +1 @@
+data = 'overridden facts.py'

test/integration/targets/module_utils/module_utils/foo.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+foo = "FOO FROM foo.py"

test/integration/targets/module_utils/module_utils/foo0.py

@@ -0,0 +1 @@
+data = 'foo0'

test/integration/targets/module_utils/module_utils/foo1.py

@@ -0,0 +1 @@
+data = 'foo1'

test/integration/targets/module_utils/module_utils/foo2.py

@@ -0,0 +1 @@
+data = 'foo2'

test/integration/targets/module_utils/module_utils/qux1/quux.py

@@ -0,0 +1 @@
+data = 'qux1'

test/integration/targets/module_utils/module_utils/qux2/quux.py

@@ -0,0 +1 @@
+data = 'qux2:quux'

test/integration/targets/module_utils/module_utils/qux2/quuz.py

@@ -0,0 +1 @@
+data = 'qux2:quuz'

test/integration/targets/module_utils/module_utils/spam1/ham/eggs/__init__.py

@@ -0,0 +1 @@
+data = 'spam1'

test/integration/targets/module_utils/module_utils/spam2/ham/eggs/__init__.py

@@ -0,0 +1 @@
+data = 'spam2'

test/integration/targets/module_utils/module_utils/spam3/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam3'

test/integration/targets/module_utils/module_utils/spam4/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam4'

test/integration/targets/module_utils/module_utils/spam5/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam5:bacon'

test/integration/targets/module_utils/module_utils/spam5/ham/eggs.py

@@ -0,0 +1 @@
+data = 'spam5:eggs'

test/integration/targets/module_utils/module_utils/spam6/ham/__init__.py

@@ -0,0 +1,2 @@
+bacon = 'spam6:bacon'
+eggs = 'spam6:eggs'

test/integration/targets/module_utils/module_utils/spam7/ham/__init__.py

@@ -0,0 +1 @@
+eggs = 'spam7:eggs'

test/integration/targets/module_utils/module_utils/spam7/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam7:bacon'

test/integration/targets/module_utils/module_utils/spam8/ham/__init__.py

@@ -0,0 +1 @@
+eggs = 'spam8:eggs'

test/integration/targets/module_utils/module_utils/spam8/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam8:bacon'

test/integration/targets/module_utils/module_utils/sub/bam.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bam = "BAM FROM sub/bam.py"

test/integration/targets/module_utils/module_utils/sub/bam/bam.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bam = "BAM FROM sub/bam/bam.py"

test/integration/targets/module_utils/module_utils/sub/bar/bam.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bam = "BAM FROM sub/bar/bam.py"

test/integration/targets/module_utils/module_utils/sub/bar/bar.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bar = "BAR FROM sub/bar/bar.py"

test/integration/targets/module_utils/module_utils/yak/zebra/foo.py

@@ -0,0 +1 @@
+data = 'yak'

test/integration/targets/module_utils/module_utils_test.yml

@@ -0,0 +1,50 @@
+- hosts: localhost
+  gather_facts: no
+  tasks:
+  - name: Use a specially crafted module to see if things were imported correctly
+    test:
+    register: result
+
+  - name: Check that the module imported the correct version of each module_util
+    assert:
+      that:
+        - 'result["abcdefgh"] == "abcdefgh"'
+        - 'result["bar0"] == "bar0"'
+        - 'result["bar1"] == "bar1"'
+        - 'result["bar2"] == "bar2"'
+        - 'result["baz1"] == "baz1"'
+        - 'result["baz2"] == "baz2"'
+        - 'result["foo0"] == "foo0"'
+        - 'result["foo1"] == "foo1"'
+        - 'result["foo2"] == "foo2"'
+        - 'result["qux1"] == "qux1"'
+        - 'result["qux2"] == ["qux2:quux", "qux2:quuz"]'
+        - 'result["spam1"] == "spam1"'
+        - 'result["spam2"] == "spam2"'
+        - 'result["spam3"] == "spam3"'
+        - 'result["spam4"] == "spam4"'
+        - 'result["spam5"] == ["spam5:bacon", "spam5:eggs"]'
+        - 'result["spam6"] == ["spam6:bacon", "spam6:eggs"]'
+        - 'result["spam7"] == ["spam7:bacon", "spam7:eggs"]'
+        - 'result["spam8"] == ["spam8:bacon", "spam8:eggs"]'
+
+  # Test that overriding something in module_utils with something in the local library works
+  - name: Test that local module_utils overrides facts.py
+    test_override:
+    register: result
+
+  - name: Make sure the we used the local facts.py, not the one shipped with ansible
+    assert:
+      that:
+        - 'result["data"] == "overridden facts.py"'
+
+  - name: Test that importing a module that only exists inside of a submodule does not work
+    test_failure:
+    ignore_errors: True
+    register: result
+
+  - name: Make sure we failed in AnsiBallZ
+    assert:
+      that:
+        - 'result["failed"] == True'
+        - '"Could not find imported module support code for test_failure.  Looked for either foo or zebra" == result["msg"]'

test/integration/targets/module_utils/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook module_utils_test.yml -i ../../inventory -v "$@"

test/sanity/code-smell/shebang.sh

@@ -6,6 +6,7 @@ grep '^#!' -rIn . \
     | grep ':1:' | sed 's/:1:/:/' | grep -v -E \
     -e '^\./lib/ansible/modules/' \
     -e '^\./test/integration/targets/[^/]*/library/[^/]*:#!powershell$' \
+    -e '^\./test/integration/targets/[^/]*/library/[^/]*:#!/usr/bin/python$' \
     -e '^\./test/sanity/validate-modules/validate-modules:#!/usr/bin/env python2$' \
     -e '^\./hacking/cherrypick.py:#!/usr/bin/env python3$' \
     -e ':#!/bin/sh$' \

test/units/executor/module_common/test_recursive_finder.py

@@ -0,0 +1,134 @@
+# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import imp
+import zipfile
+from collections import namedtuple
+from functools import partial
+from io import BytesIO, StringIO
+
+import pytest
+
+import ansible.errors
+from ansible.compat.six import PY2
+from ansible.compat.six.moves import builtins
+
+from ansible.executor.module_common import recursive_finder
+
+
+original_find_module = imp.find_module
+
+
+@pytest.fixture
+def finder_containers():
+    FinderContainers = namedtuple('FinderContainers', ['py_module_names', 'py_module_cache', 'zf'])
+
+    py_module_names = set()
+    #py_module_cache = {('__init__',): b''}
+    py_module_cache = {}
+
+    zipoutput = BytesIO()
+    zf = zipfile.ZipFile(zipoutput, mode='w', compression=zipfile.ZIP_STORED)
+    #zf.writestr('ansible/__init__.py', b'')
+
+    return FinderContainers(py_module_names, py_module_cache, zf)
+
+
+def find_module_foo(module_utils_data, *args, **kwargs):
+    if args[0] == 'foo':
+        return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo.py', ('.py', 'r', imp.PY_SOURCE))
+    return original_find_module(*args, **kwargs)
+
+
+def find_package_foo(module_utils_data, *args, **kwargs):
+    if args[0] == 'foo':
+        return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo', ('', '', imp.PKG_DIRECTORY))
+    return original_find_module(*args, **kwargs)
+
+
+class TestRecursiveFinder(object):
+    def test_no_module_utils(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nreturn \'{\"changed\": false}\''
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set(())
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset()
+
+    def test_from_import_toplevel_package(self, finder_containers, mocker):
+        if PY2:
+            module_utils_data = BytesIO(b'# License\ndef do_something():\n    pass\n')
+        else:
+            module_utils_data = StringIO(u'# License\ndef do_something():\n    pass\n')
+        mocker.patch('imp.find_module', side_effect=partial(find_package_foo, module_utils_data))
+        mocker.patch('ansible.executor.module_common._slurp', side_effect= lambda x: b'# License\ndef do_something():\n    pass\n')
+
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo'
+        recursive_finder(name, data, *finder_containers)
+        mocker.stopall()
+
+        assert finder_containers.py_module_names == set((('foo', '__init__'),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo/__init__.py',))
+
+    def test_from_import_toplevel_module(self, finder_containers, mocker):
+        if PY2:
+            module_utils_data = BytesIO(b'# License\ndef do_something():\n    pass\n')
+        else:
+            module_utils_data = StringIO(u'# License\ndef do_something():\n    pass\n')
+        mocker.patch('imp.find_module', side_effect=partial(find_module_foo, module_utils_data))
+
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo'
+        recursive_finder(name, data, *finder_containers)
+        mocker.stopall()
+
+        assert finder_containers.py_module_names == set((('foo',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo.py',))
+
+    #
+    # Test importing six with many permutations because it is not a normal module
+    #
+    def test_from_import_six(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils import six'
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
+
+    def test_import_six(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nimport ansible.module_utils.six'
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
+
+    def test_import_six_from_many_submodules(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils.six.moves.urllib.parse import urlparse'
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))