diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index d82ec2a808..eb28fffeff 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -34,6 +34,7 @@ from ansible.release import __version__, __author__ from ansible import constants as C from ansible.errors import AnsibleError from ansible.module_utils._text import to_bytes, to_text +from ansible.plugins import module_utils_loader # Must import strategy and use write_locks from there # If we import write_locks directly then we end up binding a # variable to the object and then it never gets updated. @@ -57,8 +58,8 @@ REPLACER_SELINUX = b"<>" # specify an encoding for the python source file ENCODING_STRING = u'# -*- coding: utf-8 -*-' -# we've moved the module_common relative to the snippets, so fix the path -_SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils') +# module_common is relative to module_utils, so fix the path +_MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils') # ****************************************************************************** @@ -468,13 +469,16 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): # Loop through the imports that we've found to normalize them # Exclude paths that match with paths we've already processed # (Have to exclude them a second time once the paths are processed) + + module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)] + module_utils_paths.append(_MODULE_UTILS_PATH) for py_module_name in finder.submodules.difference(py_module_names): module_info = None if py_module_name[0] == 'six': # Special case the python six library because it messes up the # import process in an incompatible way - module_info = imp.find_module('six', [_SNIPPET_PATH]) + module_info = imp.find_module('six', module_utils_paths) py_module_name = ('six',) idx = 0 else: @@ -485,7 +489,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): break try: module_info = imp.find_module(py_module_name[-idx], - [os.path.join(_SNIPPET_PATH, *py_module_name[:-idx])]) + [os.path.join(p, *py_module_name[:-idx]) for p in module_utils_paths]) break except ImportError: continue @@ -513,7 +517,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): if module_info[2][2] == imp.PKG_DIRECTORY: # Read the __init__.py instead of the module file as this is # a python package - py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(_SNIPPET_PATH, *py_module_name), '__init__.py')) + py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(module_info[1], '__init__.py'))) normalized_modules.add(py_module_name + ('__init__',)) else: py_module_cache[py_module_name] = module_info[0].read() @@ -525,8 +529,10 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): for i in range(1, len(py_module_name)): py_pkg_name = py_module_name[:-i] + ('__init__',) if py_pkg_name not in py_module_names: + pkg_dir_info = imp.find_module(py_pkg_name[-1], + [os.path.join(p, *py_pkg_name[:-1]) for p in module_utils_paths]) normalized_modules.add(py_pkg_name) - py_module_cache[py_pkg_name] = _slurp('%s.py' % os.path.join(_SNIPPET_PATH, *py_pkg_name)) + py_module_cache[py_pkg_name] = _slurp(pkg_dir_info[1]) # # iterate through all of the ansible.module_utils* imports that we haven't @@ -554,13 +560,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): del py_module_cache[py_module_file] -def _is_binary(module_data): +def _is_binary(b_module_data): textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) - start = module_data[:1024] + start = b_module_data[:1024] return bool(start.translate(None, textchars)) -def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): +def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression): """ Given the source of the module, convert it to a Jinja2 template to insert module code and return whether it's a new or old style module. @@ -574,31 +580,31 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # module_substyle is extra information that's useful internally. It tells # us what we have to look to substitute in the module files and whether # we're using module replacer or ansiballz to format the module itself. - if _is_binary(module_data): + if _is_binary(b_module_data): module_substyle = module_style = 'binary' - elif REPLACER in module_data: + elif REPLACER in b_module_data: # Do REPLACER before from ansible.module_utils because we need make sure # we substitute "from ansible.module_utils basic" for REPLACER module_style = 'new' module_substyle = 'python' - module_data = module_data.replace(REPLACER, b'from ansible.module_utils.basic import *') - elif b'from ansible.module_utils.' in module_data: + b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *') + elif b'from ansible.module_utils.' in b_module_data: module_style = 'new' module_substyle = 'python' - elif REPLACER_WINDOWS in module_data: + elif REPLACER_WINDOWS in b_module_data: module_style = 'new' module_substyle = 'powershell' - elif REPLACER_JSONARGS in module_data: + elif REPLACER_JSONARGS in b_module_data: module_style = 'new' module_substyle = 'jsonargs' - elif b'WANT_JSON' in module_data: + elif b'WANT_JSON' in b_module_data: module_substyle = module_style = 'non_native_want_json' shebang = None # Neither old-style, non_native_want_json nor binary modules should be modified # except for the shebang line (Done by modify_module) if module_style in ('old', 'non_native_want_json', 'binary'): - return module_data, module_style, shebang + return b_module_data, module_style, shebang output = BytesIO() py_module_names = set() @@ -651,10 +657,10 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta to_bytes(__author__) + b'"\n') zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n') - zf.writestr('ansible_module_%s.py' % module_name, module_data) + zf.writestr('ansible_module_%s.py' % module_name, b_module_data) py_module_cache = { ('__init__',): b'' } - recursive_finder(module_name, module_data, py_module_names, py_module_cache, zf) + recursive_finder(module_name, b_module_data, py_module_names, py_module_cache, zf) zf.close() zipdata = base64.b64encode(zipoutput.getvalue()) @@ -712,22 +718,23 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta minute=now.minute, second=now.second, ))) - module_data = output.getvalue() + b_module_data = output.getvalue() elif module_substyle == 'powershell': # Module replacer for jsonargs and windows - lines = module_data.split(b'\n') + lines = b_module_data.split(b'\n') for line in lines: if REPLACER_WINDOWS in line: - ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1")) + # FIXME: Need to make a module_utils loader for powershell at some point + ps_data = _slurp(os.path.join(_MODULE_UTILS_PATH, "powershell.ps1")) output.write(ps_data) py_module_names.add((b'powershell',)) continue output.write(line + b'\n') - module_data = output.getvalue() + b_module_data = output.getvalue() module_args_json = to_bytes(json.dumps(module_args)) - module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) + b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json) # Powershell/winrm don't actually make use of shebang so we can # safely set this here. If we let the fallback code handle this @@ -751,17 +758,17 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # ansiballz) If we remove them from jsonargs-style module replacer # then we can remove them everywhere. python_repred_args = to_bytes(repr(module_args_json)) - module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__))) - module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args) - module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS))) + b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__))) + b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args) + b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS))) # The main event -- substitute the JSON args string into the module - module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) + b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json) facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict') - module_data = module_data.replace(b'syslog.LOG_USER', facility) + b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility) - return (module_data, module_style, shebang) + return (b_module_data, module_style, shebang) def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'): @@ -791,14 +798,14 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul with open(module_path, 'rb') as f: # read in the module source - module_data = f.read() + b_module_data = f.read() - (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) + (b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression) if module_style == 'binary': - return (module_data, module_style, to_text(shebang, nonstring='passthru')) + return (b_module_data, module_style, to_text(shebang, nonstring='passthru')) elif shebang is None: - lines = module_data.split(b"\n", 1) + lines = b_module_data.split(b"\n", 1) if lines[0].startswith(b"#!"): shebang = lines[0].strip() args = shlex.split(str(shebang[2:])) @@ -815,8 +822,8 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul # No shebang, assume a binary module? pass - module_data = b"\n".join(lines) + b_module_data = b"\n".join(lines) else: shebang = to_bytes(shebang, errors='surrogate_or_strict') - return (module_data, module_style, to_text(shebang, nonstring='passthru')) + return (b_module_data, module_style, to_text(shebang, nonstring='passthru')) diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index 25152bfc00..f0d610d744 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -1,5 +1,6 @@ # (c) 2012, Daniel Hokka Zakrisson # (c) 2012-2014, Michael DeHaan and others +# (c) 2017, Toshio Kuratomi # # This file is part of Ansible # @@ -31,6 +32,7 @@ import warnings from collections import defaultdict from ansible import constants as C +from ansible.compat.six import string_types from ansible.module_utils._text import to_text @@ -148,7 +150,7 @@ class PluginLoader: results.append(os.path.join(root,x)) return results - def _get_package_paths(self): + def _get_package_paths(self, subdirs=True): ''' Gets the path of a Python package ''' if not self.package: @@ -159,11 +161,18 @@ class PluginLoader: for parent_mod in parts: m = getattr(m, parent_mod) self.package_path = os.path.dirname(m.__file__) - return self._all_directories(self.package_path) + if subdirs: + return self._all_directories(self.package_path) + return [self.package_path] - def _get_paths(self): + def _get_paths(self, subdirs=True): ''' Return a list of paths to search for plugins in ''' + # FIXME: This is potentially buggy if subdirs is sometimes True and + # sometimes False. In current usage, everything calls this with + # subdirs=True except for module_utils_loader which always calls it + # with subdirs=False. So there currently isn't a problem with this + # caching. if self._paths is not None: return self._paths @@ -173,15 +182,18 @@ class PluginLoader: if self.config is not None: for path in self.config: path = os.path.realpath(os.path.expanduser(path)) - contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path) - for c in contents: - if os.path.isdir(c) and c not in ret: - ret.append(c) + if subdirs: + contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path) + for c in contents: + if os.path.isdir(c) and c not in ret: + ret.append(c) if path not in ret: ret.append(path) # look for any plugins installed in the package subtree - ret.extend(self._get_package_paths()) + # Note package path always gets added last so that every other type of + # path is searched before it. + ret.extend(self._get_package_paths(subdirs=subdirs)) # HACK: because powershell modules are in the same directory # hierarchy as other modules we have to process them last. This is @@ -197,6 +209,7 @@ class PluginLoader: # would have class_names, they would not work as written. reordered_paths = [] win_dirs = [] + for path in ret: if path.endswith('windows'): win_dirs.append(path) @@ -260,6 +273,9 @@ class PluginLoader: # HACK: We have no way of executing python byte # compiled files as ansible modules so specifically exclude them + ### FIXME: I believe this is only correct for modules and + # module_utils. For all other plugins we want .pyc and .pyo should + # bew valid if full_path.endswith(('.pyc', '.pyo')): continue @@ -466,6 +482,13 @@ module_loader = PluginLoader( 'library', ) +module_utils_loader = PluginLoader( + '', + 'ansible.module_utils', + 'module_utils', + 'module_utils', +) + lookup_loader = PluginLoader( 'LookupModule', 'ansible.plugins.lookup', diff --git a/test/integration/targets/module_utils/aliases b/test/integration/targets/module_utils/aliases new file mode 100644 index 0000000000..79d8b9285e --- /dev/null +++ b/test/integration/targets/module_utils/aliases @@ -0,0 +1 @@ +posix/ci/group3 diff --git a/test/integration/targets/module_utils/library/test.py b/test/integration/targets/module_utils/library/test.py new file mode 100644 index 0000000000..4b09bed5e3 --- /dev/null +++ b/test/integration/targets/module_utils/library/test.py @@ -0,0 +1,83 @@ +#!/usr/bin/python + +results = {} + +# Test import with no from +import ansible.module_utils.foo0 +results['foo0'] = ansible.module_utils.foo0.data + +# Test depthful import with no from +import ansible.module_utils.bar0.foo +results['bar0'] = ansible.module_utils.bar0.foo.data + +# Test import of module_utils/foo1.py +from ansible.module_utils import foo1 +results['foo1'] = foo1.data + +# Test import of an identifier inside of module_utils/foo2.py +from ansible.module_utils.foo2 import data +results['foo2'] = data + +# Test import of module_utils/bar1/__init__.py +from ansible.module_utils import bar1 +results['bar1'] = bar1.data + +# Test import of an identifier inside of module_utils/bar2/__init__.py +from ansible.module_utils.bar2 import data +results['bar2'] = data + +# Test import of module_utils/baz1/one.py +from ansible.module_utils.baz1 import one +results['baz1'] = one.data + +# Test import of an identifier inside of module_utils/baz2/one.py +from ansible.module_utils.baz2.one import data +results['baz2'] = data + +# Test import of module_utils/spam1/ham/eggs/__init__.py +from ansible.module_utils.spam1.ham import eggs +results['spam1'] = eggs.data + +# Test import of an identifier inside module_utils/spam2/ham/eggs/__init__.py +from ansible.module_utils.spam2.ham.eggs import data +results['spam2'] = data + +# Test import of module_utils/spam3/ham/bacon.py +from ansible.module_utils.spam3.ham import bacon +results['spam3'] = bacon.data + +# Test import of an identifier inside of module_utils/spam4/ham/bacon.py +from ansible.module_utils.spam4.ham.bacon import data +results['spam4'] = data + +# Test import of module_utils.spam5.ham bacon and eggs (modules) +from ansible.module_utils.spam5.ham import bacon, eggs +results['spam5'] = (bacon.data, eggs.data) + +# Test import of module_utils.spam6.ham bacon and eggs (identifiers) +from ansible.module_utils.spam6.ham import bacon, eggs +results['spam6'] = (bacon, eggs) + +# Test import of module_utils.spam7.ham bacon and eggs (module and identifier) +from ansible.module_utils.spam7.ham import bacon, eggs +results['spam7'] = (bacon.data, eggs) + +# Test import of module_utils/spam8/ham/bacon.py and module_utils/spam8/ham/eggs.py separately +from ansible.module_utils.spam8.ham import bacon +from ansible.module_utils.spam8.ham import eggs +results['spam8'] = (bacon.data, eggs) + +# Test that import of module_utils/qux1/quux.py using as works +from ansible.module_utils.qux1 import quux as one +results['qux1'] = one.data + +# Test that importing qux2/quux.py and qux2/quuz.py using as works +from ansible.module_utils.qux2 import quux as one, quuz as two +results['qux2'] = (one.data, two.data) + +# Test depth +from ansible.module_utils.a.b.c.d.e.f.g.h import data + +results['abcdefgh'] = data +from ansible.module_utils.basic import AnsibleModule +AnsibleModule(argument_spec=dict()).exit_json(**results) diff --git a/test/integration/targets/module_utils/library/test_failure.py b/test/integration/targets/module_utils/library/test_failure.py new file mode 100644 index 0000000000..e1a87c2e71 --- /dev/null +++ b/test/integration/targets/module_utils/library/test_failure.py @@ -0,0 +1,14 @@ +#!/usr/bin/python + +results = {} +# Test that we are rooted correctly +# Following files: +# module_utils/yak/zebra/foo.py +try: + from ansible.module_utils.zebra import foo + results['zebra'] = foo.data +except ImportError: + results['zebra'] = 'Failed in module as expected but incorrectly did not fail in AnsiballZ construction' + +from ansible.module_utils.basic import AnsibleModule +AnsibleModule(argument_spec=dict()).exit_json(**results) diff --git a/test/integration/targets/module_utils/library/test_override.py b/test/integration/targets/module_utils/library/test_override.py new file mode 100644 index 0000000000..9ff54bf949 --- /dev/null +++ b/test/integration/targets/module_utils/library/test_override.py @@ -0,0 +1,7 @@ +#!/usr/bin/python +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.facts import data + +results = {"data": data} + +AnsibleModule(argument_spec=dict()).exit_json(**results) diff --git a/test/integration/targets/module_utils/module_utils/__init__.py b/test/integration/targets/module_utils/module_utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/__init__.py b/test/integration/targets/module_utils/module_utils/a/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/b/__init__.py b/test/integration/targets/module_utils/module_utils/a/b/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/b/c/__init__.py b/test/integration/targets/module_utils/module_utils/a/b/c/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/b/c/d/__init__.py b/test/integration/targets/module_utils/module_utils/a/b/c/d/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/b/c/d/e/__init__.py b/test/integration/targets/module_utils/module_utils/a/b/c/d/e/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/__init__.py b/test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/g/__init__.py b/test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/g/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/g/h/__init__.py b/test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/g/h/__init__.py new file mode 100644 index 0000000000..722f4b7750 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/g/h/__init__.py @@ -0,0 +1 @@ +data = 'abcdefgh' diff --git a/test/integration/targets/module_utils/module_utils/bar0/__init__.py b/test/integration/targets/module_utils/module_utils/bar0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/bar0/foo.py b/test/integration/targets/module_utils/module_utils/bar0/foo.py new file mode 100644 index 0000000000..1072dcc267 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/bar0/foo.py @@ -0,0 +1 @@ +data = 'bar0' diff --git a/test/integration/targets/module_utils/module_utils/bar1/__init__.py b/test/integration/targets/module_utils/module_utils/bar1/__init__.py new file mode 100644 index 0000000000..68e43509a4 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/bar1/__init__.py @@ -0,0 +1 @@ +data = 'bar1' diff --git a/test/integration/targets/module_utils/module_utils/bar2/__init__.py b/test/integration/targets/module_utils/module_utils/bar2/__init__.py new file mode 100644 index 0000000000..59e86afd5b --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/bar2/__init__.py @@ -0,0 +1 @@ +data = 'bar2' diff --git a/test/integration/targets/module_utils/module_utils/baz1/__init__.py b/test/integration/targets/module_utils/module_utils/baz1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/baz1/one.py b/test/integration/targets/module_utils/module_utils/baz1/one.py new file mode 100644 index 0000000000..e5d7894a36 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/baz1/one.py @@ -0,0 +1 @@ +data = 'baz1' diff --git a/test/integration/targets/module_utils/module_utils/baz2/__init__.py b/test/integration/targets/module_utils/module_utils/baz2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/baz2/one.py b/test/integration/targets/module_utils/module_utils/baz2/one.py new file mode 100644 index 0000000000..1efe196c3c --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/baz2/one.py @@ -0,0 +1 @@ +data = 'baz2' diff --git a/test/integration/targets/module_utils/module_utils/facts.py b/test/integration/targets/module_utils/module_utils/facts.py new file mode 100644 index 0000000000..ba7cbb7b4f --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/facts.py @@ -0,0 +1 @@ +data = 'overridden facts.py' diff --git a/test/integration/targets/module_utils/module_utils/foo.py b/test/integration/targets/module_utils/module_utils/foo.py new file mode 100644 index 0000000000..20698f1f46 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/foo.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +foo = "FOO FROM foo.py" diff --git a/test/integration/targets/module_utils/module_utils/foo0.py b/test/integration/targets/module_utils/module_utils/foo0.py new file mode 100644 index 0000000000..4b528b6d4d --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/foo0.py @@ -0,0 +1 @@ +data = 'foo0' diff --git a/test/integration/targets/module_utils/module_utils/foo1.py b/test/integration/targets/module_utils/module_utils/foo1.py new file mode 100644 index 0000000000..18e0cef14e --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/foo1.py @@ -0,0 +1 @@ +data = 'foo1' diff --git a/test/integration/targets/module_utils/module_utils/foo2.py b/test/integration/targets/module_utils/module_utils/foo2.py new file mode 100644 index 0000000000..feb142dfe1 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/foo2.py @@ -0,0 +1 @@ +data = 'foo2' diff --git a/test/integration/targets/module_utils/module_utils/qux1/__init__.py b/test/integration/targets/module_utils/module_utils/qux1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/qux1/quux.py b/test/integration/targets/module_utils/module_utils/qux1/quux.py new file mode 100644 index 0000000000..3d288c96c1 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/qux1/quux.py @@ -0,0 +1 @@ +data = 'qux1' diff --git a/test/integration/targets/module_utils/module_utils/qux2/__init__.py b/test/integration/targets/module_utils/module_utils/qux2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/qux2/quux.py b/test/integration/targets/module_utils/module_utils/qux2/quux.py new file mode 100644 index 0000000000..496d446aff --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/qux2/quux.py @@ -0,0 +1 @@ +data = 'qux2:quux' diff --git a/test/integration/targets/module_utils/module_utils/qux2/quuz.py b/test/integration/targets/module_utils/module_utils/qux2/quuz.py new file mode 100644 index 0000000000..cdc0fad75f --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/qux2/quuz.py @@ -0,0 +1 @@ +data = 'qux2:quuz' diff --git a/test/integration/targets/module_utils/module_utils/spam1/__init__.py b/test/integration/targets/module_utils/module_utils/spam1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam1/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam1/ham/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam1/ham/eggs/__init__.py b/test/integration/targets/module_utils/module_utils/spam1/ham/eggs/__init__.py new file mode 100644 index 0000000000..f290e156f6 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam1/ham/eggs/__init__.py @@ -0,0 +1 @@ +data = 'spam1' diff --git a/test/integration/targets/module_utils/module_utils/spam2/__init__.py b/test/integration/targets/module_utils/module_utils/spam2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam2/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam2/ham/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam2/ham/eggs/__init__.py b/test/integration/targets/module_utils/module_utils/spam2/ham/eggs/__init__.py new file mode 100644 index 0000000000..5e053d88ed --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam2/ham/eggs/__init__.py @@ -0,0 +1 @@ +data = 'spam2' diff --git a/test/integration/targets/module_utils/module_utils/spam3/__init__.py b/test/integration/targets/module_utils/module_utils/spam3/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam3/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam3/ham/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam3/ham/bacon.py b/test/integration/targets/module_utils/module_utils/spam3/ham/bacon.py new file mode 100644 index 0000000000..9107508973 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam3/ham/bacon.py @@ -0,0 +1 @@ +data = 'spam3' diff --git a/test/integration/targets/module_utils/module_utils/spam4/__init__.py b/test/integration/targets/module_utils/module_utils/spam4/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam4/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam4/ham/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam4/ham/bacon.py b/test/integration/targets/module_utils/module_utils/spam4/ham/bacon.py new file mode 100644 index 0000000000..7d55288211 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam4/ham/bacon.py @@ -0,0 +1 @@ +data = 'spam4' diff --git a/test/integration/targets/module_utils/module_utils/spam5/__init__.py b/test/integration/targets/module_utils/module_utils/spam5/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam5/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam5/ham/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam5/ham/bacon.py b/test/integration/targets/module_utils/module_utils/spam5/ham/bacon.py new file mode 100644 index 0000000000..cc947b83c5 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam5/ham/bacon.py @@ -0,0 +1 @@ +data = 'spam5:bacon' diff --git a/test/integration/targets/module_utils/module_utils/spam5/ham/eggs.py b/test/integration/targets/module_utils/module_utils/spam5/ham/eggs.py new file mode 100644 index 0000000000..f0394c8777 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam5/ham/eggs.py @@ -0,0 +1 @@ +data = 'spam5:eggs' diff --git a/test/integration/targets/module_utils/module_utils/spam6/__init__.py b/test/integration/targets/module_utils/module_utils/spam6/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam6/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam6/ham/__init__.py new file mode 100644 index 0000000000..8c1a70eaba --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam6/ham/__init__.py @@ -0,0 +1,2 @@ +bacon = 'spam6:bacon' +eggs = 'spam6:eggs' diff --git a/test/integration/targets/module_utils/module_utils/spam7/__init__.py b/test/integration/targets/module_utils/module_utils/spam7/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam7/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam7/ham/__init__.py new file mode 100644 index 0000000000..cd9a05d0a4 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam7/ham/__init__.py @@ -0,0 +1 @@ +eggs = 'spam7:eggs' diff --git a/test/integration/targets/module_utils/module_utils/spam7/ham/bacon.py b/test/integration/targets/module_utils/module_utils/spam7/ham/bacon.py new file mode 100644 index 0000000000..490121f897 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam7/ham/bacon.py @@ -0,0 +1 @@ +data = 'spam7:bacon' diff --git a/test/integration/targets/module_utils/module_utils/spam8/__init__.py b/test/integration/targets/module_utils/module_utils/spam8/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/spam8/ham/__init__.py b/test/integration/targets/module_utils/module_utils/spam8/ham/__init__.py new file mode 100644 index 0000000000..c02bf5fdb4 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam8/ham/__init__.py @@ -0,0 +1 @@ +eggs = 'spam8:eggs' diff --git a/test/integration/targets/module_utils/module_utils/spam8/ham/bacon.py b/test/integration/targets/module_utils/module_utils/spam8/ham/bacon.py new file mode 100644 index 0000000000..28ea2857f3 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/spam8/ham/bacon.py @@ -0,0 +1 @@ +data = 'spam8:bacon' diff --git a/test/integration/targets/module_utils/module_utils/sub/__init__.py b/test/integration/targets/module_utils/module_utils/sub/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/sub/bam.py b/test/integration/targets/module_utils/module_utils/sub/bam.py new file mode 100644 index 0000000000..566f8b7c47 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/sub/bam.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +bam = "BAM FROM sub/bam.py" diff --git a/test/integration/targets/module_utils/module_utils/sub/bam/__init__.py b/test/integration/targets/module_utils/module_utils/sub/bam/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/sub/bam/bam.py b/test/integration/targets/module_utils/module_utils/sub/bam/bam.py new file mode 100644 index 0000000000..b7ed707211 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/sub/bam/bam.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +bam = "BAM FROM sub/bam/bam.py" diff --git a/test/integration/targets/module_utils/module_utils/sub/bar/__init__.py b/test/integration/targets/module_utils/module_utils/sub/bar/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/sub/bar/bam.py b/test/integration/targets/module_utils/module_utils/sub/bar/bam.py new file mode 100644 index 0000000000..02fafd40f3 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/sub/bar/bam.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +bam = "BAM FROM sub/bar/bam.py" diff --git a/test/integration/targets/module_utils/module_utils/sub/bar/bar.py b/test/integration/targets/module_utils/module_utils/sub/bar/bar.py new file mode 100644 index 0000000000..8566901f04 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/sub/bar/bar.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +bar = "BAR FROM sub/bar/bar.py" diff --git a/test/integration/targets/module_utils/module_utils/yak/__init__.py b/test/integration/targets/module_utils/module_utils/yak/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/yak/zebra/__init__.py b/test/integration/targets/module_utils/module_utils/yak/zebra/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/module_utils/module_utils/yak/zebra/foo.py b/test/integration/targets/module_utils/module_utils/yak/zebra/foo.py new file mode 100644 index 0000000000..89b2bfe8c6 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils/yak/zebra/foo.py @@ -0,0 +1 @@ +data = 'yak' diff --git a/test/integration/targets/module_utils/module_utils_test.yml b/test/integration/targets/module_utils/module_utils_test.yml new file mode 100644 index 0000000000..8a5cccce47 --- /dev/null +++ b/test/integration/targets/module_utils/module_utils_test.yml @@ -0,0 +1,50 @@ +- hosts: localhost + gather_facts: no + tasks: + - name: Use a specially crafted module to see if things were imported correctly + test: + register: result + + - name: Check that the module imported the correct version of each module_util + assert: + that: + - 'result["abcdefgh"] == "abcdefgh"' + - 'result["bar0"] == "bar0"' + - 'result["bar1"] == "bar1"' + - 'result["bar2"] == "bar2"' + - 'result["baz1"] == "baz1"' + - 'result["baz2"] == "baz2"' + - 'result["foo0"] == "foo0"' + - 'result["foo1"] == "foo1"' + - 'result["foo2"] == "foo2"' + - 'result["qux1"] == "qux1"' + - 'result["qux2"] == ["qux2:quux", "qux2:quuz"]' + - 'result["spam1"] == "spam1"' + - 'result["spam2"] == "spam2"' + - 'result["spam3"] == "spam3"' + - 'result["spam4"] == "spam4"' + - 'result["spam5"] == ["spam5:bacon", "spam5:eggs"]' + - 'result["spam6"] == ["spam6:bacon", "spam6:eggs"]' + - 'result["spam7"] == ["spam7:bacon", "spam7:eggs"]' + - 'result["spam8"] == ["spam8:bacon", "spam8:eggs"]' + + # Test that overriding something in module_utils with something in the local library works + - name: Test that local module_utils overrides facts.py + test_override: + register: result + + - name: Make sure the we used the local facts.py, not the one shipped with ansible + assert: + that: + - 'result["data"] == "overridden facts.py"' + + - name: Test that importing a module that only exists inside of a submodule does not work + test_failure: + ignore_errors: True + register: result + + - name: Make sure we failed in AnsiBallZ + assert: + that: + - 'result["failed"] == True' + - '"Could not find imported module support code for test_failure. Looked for either foo or zebra" == result["msg"]' diff --git a/test/integration/targets/module_utils/runme.sh b/test/integration/targets/module_utils/runme.sh new file mode 100755 index 0000000000..ddde89f10a --- /dev/null +++ b/test/integration/targets/module_utils/runme.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eux + +ansible-playbook module_utils_test.yml -i ../../inventory -v "$@" diff --git a/test/sanity/code-smell/shebang.sh b/test/sanity/code-smell/shebang.sh index b731ce7459..6b1c07f067 100755 --- a/test/sanity/code-smell/shebang.sh +++ b/test/sanity/code-smell/shebang.sh @@ -6,6 +6,7 @@ grep '^#!' -rIn . \ | grep ':1:' | sed 's/:1:/:/' | grep -v -E \ -e '^\./lib/ansible/modules/' \ -e '^\./test/integration/targets/[^/]*/library/[^/]*:#!powershell$' \ + -e '^\./test/integration/targets/[^/]*/library/[^/]*:#!/usr/bin/python$' \ -e '^\./test/sanity/validate-modules/validate-modules:#!/usr/bin/env python2$' \ -e '^\./hacking/cherrypick.py:#!/usr/bin/env python3$' \ -e ':#!/bin/sh$' \ diff --git a/test/units/executor/module_common/test_recursive_finder.py b/test/units/executor/module_common/test_recursive_finder.py new file mode 100644 index 0000000000..aeb9ca13bd --- /dev/null +++ b/test/units/executor/module_common/test_recursive_finder.py @@ -0,0 +1,134 @@ +# (c) 2017, Toshio Kuratomi +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import imp +import zipfile +from collections import namedtuple +from functools import partial +from io import BytesIO, StringIO + +import pytest + +import ansible.errors +from ansible.compat.six import PY2 +from ansible.compat.six.moves import builtins + +from ansible.executor.module_common import recursive_finder + + +original_find_module = imp.find_module + + +@pytest.fixture +def finder_containers(): + FinderContainers = namedtuple('FinderContainers', ['py_module_names', 'py_module_cache', 'zf']) + + py_module_names = set() + #py_module_cache = {('__init__',): b''} + py_module_cache = {} + + zipoutput = BytesIO() + zf = zipfile.ZipFile(zipoutput, mode='w', compression=zipfile.ZIP_STORED) + #zf.writestr('ansible/__init__.py', b'') + + return FinderContainers(py_module_names, py_module_cache, zf) + + +def find_module_foo(module_utils_data, *args, **kwargs): + if args[0] == 'foo': + return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo.py', ('.py', 'r', imp.PY_SOURCE)) + return original_find_module(*args, **kwargs) + + +def find_package_foo(module_utils_data, *args, **kwargs): + if args[0] == 'foo': + return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo', ('', '', imp.PKG_DIRECTORY)) + return original_find_module(*args, **kwargs) + + +class TestRecursiveFinder(object): + def test_no_module_utils(self, finder_containers): + name = 'ping' + data = b'#!/usr/bin/python\nreturn \'{\"changed\": false}\'' + recursive_finder(name, data, *finder_containers) + assert finder_containers.py_module_names == set(()) + assert finder_containers.py_module_cache == {} + assert frozenset(finder_containers.zf.namelist()) == frozenset() + + def test_from_import_toplevel_package(self, finder_containers, mocker): + if PY2: + module_utils_data = BytesIO(b'# License\ndef do_something():\n pass\n') + else: + module_utils_data = StringIO(u'# License\ndef do_something():\n pass\n') + mocker.patch('imp.find_module', side_effect=partial(find_package_foo, module_utils_data)) + mocker.patch('ansible.executor.module_common._slurp', side_effect= lambda x: b'# License\ndef do_something():\n pass\n') + + name = 'ping' + data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo' + recursive_finder(name, data, *finder_containers) + mocker.stopall() + + assert finder_containers.py_module_names == set((('foo', '__init__'),)) + assert finder_containers.py_module_cache == {} + assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo/__init__.py',)) + + def test_from_import_toplevel_module(self, finder_containers, mocker): + if PY2: + module_utils_data = BytesIO(b'# License\ndef do_something():\n pass\n') + else: + module_utils_data = StringIO(u'# License\ndef do_something():\n pass\n') + mocker.patch('imp.find_module', side_effect=partial(find_module_foo, module_utils_data)) + + name = 'ping' + data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo' + recursive_finder(name, data, *finder_containers) + mocker.stopall() + + assert finder_containers.py_module_names == set((('foo',),)) + assert finder_containers.py_module_cache == {} + assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo.py',)) + + # + # Test importing six with many permutations because it is not a normal module + # + def test_from_import_six(self, finder_containers): + name = 'ping' + data = b'#!/usr/bin/python\nfrom ansible.module_utils import six' + recursive_finder(name, data, *finder_containers) + assert finder_containers.py_module_names == set((('six',),)) + assert finder_containers.py_module_cache == {} + assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',)) + + def test_import_six(self, finder_containers): + name = 'ping' + data = b'#!/usr/bin/python\nimport ansible.module_utils.six' + recursive_finder(name, data, *finder_containers) + assert finder_containers.py_module_names == set((('six',),)) + assert finder_containers.py_module_cache == {} + assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',)) + + def test_import_six_from_many_submodules(self, finder_containers): + name = 'ping' + data = b'#!/usr/bin/python\nfrom ansible.module_utils.six.moves.urllib.parse import urlparse' + recursive_finder(name, data, *finder_containers) + assert finder_containers.py_module_names == set((('six',),)) + assert finder_containers.py_module_cache == {} + assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))