mirror of
https://github.com/ansible-collections/community.general.git
synced 2024-09-14 20:13:21 +02:00
Combine jimi-c and bcoca's ideas and work on hooking module-utils into PluginLoader.
This version just gets the relevant paths from PluginLoader and then uses the existing imp.find_plugin() calls in the AnsiballZ code to load the proper module_utils. Modify PluginLoader to optionally omit subdirectories (module_utils needs to operate on top level dirs, not on subdirs because it has a hierarchical namespace whereas all other plugins use a flat namespace). Rename snippet* variables to module_utils* Add a small number of unittests for recursive_finder Add a larger number of integration tests to demonstrate that module_utils is working. Whitelist module-style shebang in test target library dirs Prefix module_data variable with b_ to be clear that it holds bytes data
This commit is contained in:
Toshio Kuratomi
parent
b89f222028
commit
5c38f3cea2
71 changed files with 410 additions and 44 deletions
|
|
@ -34,6 +34,7 @@ from ansible.release import __version__, __author__
|
|||
from ansible import constants as C
|
||||
from ansible.errors import AnsibleError
|
||||
from ansible.module_utils._text import to_bytes, to_text
|
||||
from ansible.plugins import module_utils_loader
|
||||
# Must import strategy and use write_locks from there
|
||||
# If we import write_locks directly then we end up binding a
|
||||
# variable to the object and then it never gets updated.
|
||||
|
|
@ -57,8 +58,8 @@ REPLACER_SELINUX = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"
|
|||
# specify an encoding for the python source file
|
||||
ENCODING_STRING = u'# -*- coding: utf-8 -*-'
|
||||
|
||||
# we've moved the module_common relative to the snippets, so fix the path
|
||||
_SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
|
||||
# module_common is relative to module_utils, so fix the path
|
||||
_MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
|
||||
|
||||
# ******************************************************************************
|
||||
|
||||
|
|
@ -468,13 +469,16 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
|
|||
# Loop through the imports that we've found to normalize them
|
||||
# Exclude paths that match with paths we've already processed
|
||||
# (Have to exclude them a second time once the paths are processed)
|
||||
|
||||
module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)]
|
||||
module_utils_paths.append(_MODULE_UTILS_PATH)
|
||||
for py_module_name in finder.submodules.difference(py_module_names):
|
||||
module_info = None
|
||||
|
||||
if py_module_name[0] == 'six':
|
||||
# Special case the python six library because it messes up the
|
||||
# import process in an incompatible way
|
||||
module_info = imp.find_module('six', [_SNIPPET_PATH])
|
||||
module_info = imp.find_module('six', module_utils_paths)
|
||||
py_module_name = ('six',)
|
||||
idx = 0
|
||||
else:
|
||||
|
|
@ -485,7 +489,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
|
|||
break
|
||||
try:
|
||||
module_info = imp.find_module(py_module_name[-idx],
|
||||
[os.path.join(_SNIPPET_PATH, *py_module_name[:-idx])])
|
||||
[os.path.join(p, *py_module_name[:-idx]) for p in module_utils_paths])
|
||||
break
|
||||
except ImportError:
|
||||
continue
|
||||
|
|
@ -513,7 +517,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
|
|||
if module_info[2][2] == imp.PKG_DIRECTORY:
|
||||
# Read the __init__.py instead of the module file as this is
|
||||
# a python package
|
||||
py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(_SNIPPET_PATH, *py_module_name), '__init__.py'))
|
||||
py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(module_info[1], '__init__.py')))
|
||||
normalized_modules.add(py_module_name + ('__init__',))
|
||||
else:
|
||||
py_module_cache[py_module_name] = module_info[0].read()
|
||||
|
|
@ -525,8 +529,10 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
|
|||
for i in range(1, len(py_module_name)):
|
||||
py_pkg_name = py_module_name[:-i] + ('__init__',)
|
||||
if py_pkg_name not in py_module_names:
|
||||
pkg_dir_info = imp.find_module(py_pkg_name[-1],
|
||||
[os.path.join(p, *py_pkg_name[:-1]) for p in module_utils_paths])
|
||||
normalized_modules.add(py_pkg_name)
|
||||
py_module_cache[py_pkg_name] = _slurp('%s.py' % os.path.join(_SNIPPET_PATH, *py_pkg_name))
|
||||
py_module_cache[py_pkg_name] = _slurp(pkg_dir_info[1])
|
||||
|
||||
#
|
||||
# iterate through all of the ansible.module_utils* imports that we haven't
|
||||
|
|
@ -554,13 +560,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
|
|||
del py_module_cache[py_module_file]
|
||||
|
||||
|
||||
def _is_binary(module_data):
|
||||
def _is_binary(b_module_data):
|
||||
textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
|
||||
start = module_data[:1024]
|
||||
start = b_module_data[:1024]
|
||||
return bool(start.translate(None, textchars))
|
||||
|
||||
|
||||
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
|
||||
def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression):
|
||||
"""
|
||||
Given the source of the module, convert it to a Jinja2 template to insert
|
||||
module code and return whether it's a new or old style module.
|
||||
|
|
@ -574,31 +580,31 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
|
|||
# module_substyle is extra information that's useful internally. It tells
|
||||
# us what we have to look to substitute in the module files and whether
|
||||
# we're using module replacer or ansiballz to format the module itself.
|
||||
if _is_binary(module_data):
|
||||
if _is_binary(b_module_data):
|
||||
module_substyle = module_style = 'binary'
|
||||
elif REPLACER in module_data:
|
||||
elif REPLACER in b_module_data:
|
||||
# Do REPLACER before from ansible.module_utils because we need make sure
|
||||
# we substitute "from ansible.module_utils basic" for REPLACER
|
||||
module_style = 'new'
|
||||
module_substyle = 'python'
|
||||
module_data = module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
|
||||
elif b'from ansible.module_utils.' in module_data:
|
||||
b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
|
||||
elif b'from ansible.module_utils.' in b_module_data:
|
||||
module_style = 'new'
|
||||
module_substyle = 'python'
|
||||
elif REPLACER_WINDOWS in module_data:
|
||||
elif REPLACER_WINDOWS in b_module_data:
|
||||
module_style = 'new'
|
||||
module_substyle = 'powershell'
|
||||
elif REPLACER_JSONARGS in module_data:
|
||||
elif REPLACER_JSONARGS in b_module_data:
|
||||
module_style = 'new'
|
||||
module_substyle = 'jsonargs'
|
||||
elif b'WANT_JSON' in module_data:
|
||||
elif b'WANT_JSON' in b_module_data:
|
||||
module_substyle = module_style = 'non_native_want_json'
|
||||
|
||||
shebang = None
|
||||
# Neither old-style, non_native_want_json nor binary modules should be modified
|
||||
# except for the shebang line (Done by modify_module)
|
||||
if module_style in ('old', 'non_native_want_json', 'binary'):
|
||||
return module_data, module_style, shebang
|
||||
return b_module_data, module_style, shebang
|
||||
|
||||
output = BytesIO()
|
||||
py_module_names = set()
|
||||
|
|
@ -651,10 +657,10 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
|
|||
to_bytes(__author__) + b'"\n')
|
||||
zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n')
|
||||
|
||||
zf.writestr('ansible_module_%s.py' % module_name, module_data)
|
||||
zf.writestr('ansible_module_%s.py' % module_name, b_module_data)
|
||||
|
||||
py_module_cache = { ('__init__',): b'' }
|
||||
recursive_finder(module_name, module_data, py_module_names, py_module_cache, zf)
|
||||
recursive_finder(module_name, b_module_data, py_module_names, py_module_cache, zf)
|
||||
zf.close()
|
||||
zipdata = base64.b64encode(zipoutput.getvalue())
|
||||
|
||||
|
|
@ -712,22 +718,23 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
|
|||
minute=now.minute,
|
||||
second=now.second,
|
||||
)))
|
||||
module_data = output.getvalue()
|
||||
b_module_data = output.getvalue()
|
||||
|
||||
elif module_substyle == 'powershell':
|
||||
# Module replacer for jsonargs and windows
|
||||
lines = module_data.split(b'\n')
|
||||
lines = b_module_data.split(b'\n')
|
||||
for line in lines:
|
||||
if REPLACER_WINDOWS in line:
|
||||
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
|
||||
# FIXME: Need to make a module_utils loader for powershell at some point
|
||||
ps_data = _slurp(os.path.join(_MODULE_UTILS_PATH, "powershell.ps1"))
|
||||
output.write(ps_data)
|
||||
py_module_names.add((b'powershell',))
|
||||
continue
|
||||
output.write(line + b'\n')
|
||||
module_data = output.getvalue()
|
||||
b_module_data = output.getvalue()
|
||||
|
||||
module_args_json = to_bytes(json.dumps(module_args))
|
||||
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
|
||||
b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)
|
||||
|
||||
# Powershell/winrm don't actually make use of shebang so we can
|
||||
# safely set this here. If we let the fallback code handle this
|
||||
|
|
@ -751,17 +758,17 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
|
|||
# ansiballz) If we remove them from jsonargs-style module replacer
|
||||
# then we can remove them everywhere.
|
||||
python_repred_args = to_bytes(repr(module_args_json))
|
||||
module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
|
||||
module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
|
||||
module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
|
||||
b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
|
||||
b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args)
|
||||
b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
|
||||
|
||||
# The main event -- substitute the JSON args string into the module
|
||||
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
|
||||
b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)
|
||||
|
||||
facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict')
|
||||
module_data = module_data.replace(b'syslog.LOG_USER', facility)
|
||||
b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility)
|
||||
|
||||
return (module_data, module_style, shebang)
|
||||
return (b_module_data, module_style, shebang)
|
||||
|
||||
|
||||
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
|
||||
|
|
@ -791,14 +798,14 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
|
|||
with open(module_path, 'rb') as f:
|
||||
|
||||
# read in the module source
|
||||
module_data = f.read()
|
||||
b_module_data = f.read()
|
||||
|
||||
(module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
|
||||
(b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression)
|
||||
|
||||
if module_style == 'binary':
|
||||
return (module_data, module_style, to_text(shebang, nonstring='passthru'))
|
||||
return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))
|
||||
elif shebang is None:
|
||||
lines = module_data.split(b"\n", 1)
|
||||
lines = b_module_data.split(b"\n", 1)
|
||||
if lines[0].startswith(b"#!"):
|
||||
shebang = lines[0].strip()
|
||||
args = shlex.split(str(shebang[2:]))
|
||||
|
|
@ -815,8 +822,8 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
|
|||
# No shebang, assume a binary module?
|
||||
pass
|
||||
|
||||
module_data = b"\n".join(lines)
|
||||
b_module_data = b"\n".join(lines)
|
||||
else:
|
||||
shebang = to_bytes(shebang, errors='surrogate_or_strict')
|
||||
|
||||
return (module_data, module_style, to_text(shebang, nonstring='passthru'))
|
||||
return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
|
||||
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> and others
|
||||
# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com>
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
|
|
@ -31,6 +32,7 @@ import warnings
|
|||
from collections import defaultdict
|
||||
|
||||
from ansible import constants as C
|
||||
from ansible.compat.six import string_types
|
||||
from ansible.module_utils._text import to_text
|
||||
|
||||
|
||||
|
|
@ -148,7 +150,7 @@ class PluginLoader:
|
|||
results.append(os.path.join(root,x))
|
||||
return results
|
||||
|
||||
def _get_package_paths(self):
|
||||
def _get_package_paths(self, subdirs=True):
|
||||
''' Gets the path of a Python package '''
|
||||
|
||||
if not self.package:
|
||||
|
|
@ -159,11 +161,18 @@ class PluginLoader:
|
|||
for parent_mod in parts:
|
||||
m = getattr(m, parent_mod)
|
||||
self.package_path = os.path.dirname(m.__file__)
|
||||
return self._all_directories(self.package_path)
|
||||
if subdirs:
|
||||
return self._all_directories(self.package_path)
|
||||
return [self.package_path]
|
||||
|
||||
def _get_paths(self):
|
||||
def _get_paths(self, subdirs=True):
|
||||
''' Return a list of paths to search for plugins in '''
|
||||
|
||||
# FIXME: This is potentially buggy if subdirs is sometimes True and
|
||||
# sometimes False. In current usage, everything calls this with
|
||||
# subdirs=True except for module_utils_loader which always calls it
|
||||
# with subdirs=False. So there currently isn't a problem with this
|
||||
# caching.
|
||||
if self._paths is not None:
|
||||
return self._paths
|
||||
|
||||
|
|
@ -173,15 +182,18 @@ class PluginLoader:
|
|||
if self.config is not None:
|
||||
for path in self.config:
|
||||
path = os.path.realpath(os.path.expanduser(path))
|
||||
contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path)
|
||||
for c in contents:
|
||||
if os.path.isdir(c) and c not in ret:
|
||||
ret.append(c)
|
||||
if subdirs:
|
||||
contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path)
|
||||
for c in contents:
|
||||
if os.path.isdir(c) and c not in ret:
|
||||
ret.append(c)
|
||||
if path not in ret:
|
||||
ret.append(path)
|
||||
|
||||
# look for any plugins installed in the package subtree
|
||||
ret.extend(self._get_package_paths())
|
||||
# Note package path always gets added last so that every other type of
|
||||
# path is searched before it.
|
||||
ret.extend(self._get_package_paths(subdirs=subdirs))
|
||||
|
||||
# HACK: because powershell modules are in the same directory
|
||||
# hierarchy as other modules we have to process them last. This is
|
||||
|
|
@ -197,6 +209,7 @@ class PluginLoader:
|
|||
# would have class_names, they would not work as written.
|
||||
reordered_paths = []
|
||||
win_dirs = []
|
||||
|
||||
for path in ret:
|
||||
if path.endswith('windows'):
|
||||
win_dirs.append(path)
|
||||
|
|
@ -260,6 +273,9 @@ class PluginLoader:
|
|||
|
||||
# HACK: We have no way of executing python byte
|
||||
# compiled files as ansible modules so specifically exclude them
|
||||
### FIXME: I believe this is only correct for modules and
|
||||
# module_utils. For all other plugins we want .pyc and .pyo should
|
||||
# bew valid
|
||||
if full_path.endswith(('.pyc', '.pyo')):
|
||||
continue
|
||||
|
||||
|
|
@ -466,6 +482,13 @@ module_loader = PluginLoader(
|
|||
'library',
|
||||
)
|
||||
|
||||
module_utils_loader = PluginLoader(
|
||||
'',
|
||||
'ansible.module_utils',
|
||||
'module_utils',
|
||||
'module_utils',
|
||||
)
|
||||
|
||||
lookup_loader = PluginLoader(
|
||||
'LookupModule',
|
||||
'ansible.plugins.lookup',
|
||||
|
|
|
|||
1
test/integration/targets/module_utils/aliases
Normal file
1
test/integration/targets/module_utils/aliases
Normal file
|
|
@ -0,0 +1 @@
|
|||
posix/ci/group3
|
||||
83
test/integration/targets/module_utils/library/test.py
Normal file
83
test/integration/targets/module_utils/library/test.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
results = {}
|
||||
|
||||
# Test import with no from
|
||||
import ansible.module_utils.foo0
|
||||
results['foo0'] = ansible.module_utils.foo0.data
|
||||
|
||||
# Test depthful import with no from
|
||||
import ansible.module_utils.bar0.foo
|
||||
results['bar0'] = ansible.module_utils.bar0.foo.data
|
||||
|
||||
# Test import of module_utils/foo1.py
|
||||
from ansible.module_utils import foo1
|
||||
results['foo1'] = foo1.data
|
||||
|
||||
# Test import of an identifier inside of module_utils/foo2.py
|
||||
from ansible.module_utils.foo2 import data
|
||||
results['foo2'] = data
|
||||
|
||||
# Test import of module_utils/bar1/__init__.py
|
||||
from ansible.module_utils import bar1
|
||||
results['bar1'] = bar1.data
|
||||
|
||||
# Test import of an identifier inside of module_utils/bar2/__init__.py
|
||||
from ansible.module_utils.bar2 import data
|
||||
results['bar2'] = data
|
||||
|
||||
# Test import of module_utils/baz1/one.py
|
||||
from ansible.module_utils.baz1 import one
|
||||
results['baz1'] = one.data
|
||||
|
||||
# Test import of an identifier inside of module_utils/baz2/one.py
|
||||
from ansible.module_utils.baz2.one import data
|
||||
results['baz2'] = data
|
||||
|
||||
# Test import of module_utils/spam1/ham/eggs/__init__.py
|
||||
from ansible.module_utils.spam1.ham import eggs
|
||||
results['spam1'] = eggs.data
|
||||
|
||||
# Test import of an identifier inside module_utils/spam2/ham/eggs/__init__.py
|
||||
from ansible.module_utils.spam2.ham.eggs import data
|
||||
results['spam2'] = data
|
||||
|
||||
# Test import of module_utils/spam3/ham/bacon.py
|
||||
from ansible.module_utils.spam3.ham import bacon
|
||||
results['spam3'] = bacon.data
|
||||
|
||||
# Test import of an identifier inside of module_utils/spam4/ham/bacon.py
|
||||
from ansible.module_utils.spam4.ham.bacon import data
|
||||
results['spam4'] = data
|
||||
|
||||
# Test import of module_utils.spam5.ham bacon and eggs (modules)
|
||||
from ansible.module_utils.spam5.ham import bacon, eggs
|
||||
results['spam5'] = (bacon.data, eggs.data)
|
||||
|
||||
# Test import of module_utils.spam6.ham bacon and eggs (identifiers)
|
||||
from ansible.module_utils.spam6.ham import bacon, eggs
|
||||
results['spam6'] = (bacon, eggs)
|
||||
|
||||
# Test import of module_utils.spam7.ham bacon and eggs (module and identifier)
|
||||
from ansible.module_utils.spam7.ham import bacon, eggs
|
||||
results['spam7'] = (bacon.data, eggs)
|
||||
|
||||
# Test import of module_utils/spam8/ham/bacon.py and module_utils/spam8/ham/eggs.py separately
|
||||
from ansible.module_utils.spam8.ham import bacon
|
||||
from ansible.module_utils.spam8.ham import eggs
|
||||
results['spam8'] = (bacon.data, eggs)
|
||||
|
||||
# Test that import of module_utils/qux1/quux.py using as works
|
||||
from ansible.module_utils.qux1 import quux as one
|
||||
results['qux1'] = one.data
|
||||
|
||||
# Test that importing qux2/quux.py and qux2/quuz.py using as works
|
||||
from ansible.module_utils.qux2 import quux as one, quuz as two
|
||||
results['qux2'] = (one.data, two.data)
|
||||
|
||||
# Test depth
|
||||
from ansible.module_utils.a.b.c.d.e.f.g.h import data
|
||||
|
||||
results['abcdefgh'] = data
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
AnsibleModule(argument_spec=dict()).exit_json(**results)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
results = {}
|
||||
# Test that we are rooted correctly
|
||||
# Following files:
|
||||
# module_utils/yak/zebra/foo.py
|
||||
try:
|
||||
from ansible.module_utils.zebra import foo
|
||||
results['zebra'] = foo.data
|
||||
except ImportError:
|
||||
results['zebra'] = 'Failed in module as expected but incorrectly did not fail in AnsiballZ construction'
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
AnsibleModule(argument_spec=dict()).exit_json(**results)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/python
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
from ansible.module_utils.facts import data
|
||||
|
||||
results = {"data": data}
|
||||
|
||||
AnsibleModule(argument_spec=dict()).exit_json(**results)
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'abcdefgh'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'bar0'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'bar1'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'bar2'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'baz1'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'baz2'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'overridden facts.py'
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
foo = "FOO FROM foo.py"
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'foo0'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'foo1'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'foo2'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'qux1'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'qux2:quux'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'qux2:quuz'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam1'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam2'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam3'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam4'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam5:bacon'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam5:eggs'
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
bacon = 'spam6:bacon'
|
||||
eggs = 'spam6:eggs'
|
||||
|
|
@ -0,0 +1 @@
|
|||
eggs = 'spam7:eggs'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam7:bacon'
|
||||
|
|
@ -0,0 +1 @@
|
|||
eggs = 'spam8:eggs'
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'spam8:bacon'
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
bam = "BAM FROM sub/bam.py"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
bam = "BAM FROM sub/bam/bam.py"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
bam = "BAM FROM sub/bar/bam.py"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
bar = "BAR FROM sub/bar/bar.py"
|
||||
|
|
@ -0,0 +1 @@
|
|||
data = 'yak'
|
||||
50
test/integration/targets/module_utils/module_utils_test.yml
Normal file
50
test/integration/targets/module_utils/module_utils_test.yml
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
- hosts: localhost
|
||||
gather_facts: no
|
||||
tasks:
|
||||
- name: Use a specially crafted module to see if things were imported correctly
|
||||
test:
|
||||
register: result
|
||||
|
||||
- name: Check that the module imported the correct version of each module_util
|
||||
assert:
|
||||
that:
|
||||
- 'result["abcdefgh"] == "abcdefgh"'
|
||||
- 'result["bar0"] == "bar0"'
|
||||
- 'result["bar1"] == "bar1"'
|
||||
- 'result["bar2"] == "bar2"'
|
||||
- 'result["baz1"] == "baz1"'
|
||||
- 'result["baz2"] == "baz2"'
|
||||
- 'result["foo0"] == "foo0"'
|
||||
- 'result["foo1"] == "foo1"'
|
||||
- 'result["foo2"] == "foo2"'
|
||||
- 'result["qux1"] == "qux1"'
|
||||
- 'result["qux2"] == ["qux2:quux", "qux2:quuz"]'
|
||||
- 'result["spam1"] == "spam1"'
|
||||
- 'result["spam2"] == "spam2"'
|
||||
- 'result["spam3"] == "spam3"'
|
||||
- 'result["spam4"] == "spam4"'
|
||||
- 'result["spam5"] == ["spam5:bacon", "spam5:eggs"]'
|
||||
- 'result["spam6"] == ["spam6:bacon", "spam6:eggs"]'
|
||||
- 'result["spam7"] == ["spam7:bacon", "spam7:eggs"]'
|
||||
- 'result["spam8"] == ["spam8:bacon", "spam8:eggs"]'
|
||||
|
||||
# Test that overriding something in module_utils with something in the local library works
|
||||
- name: Test that local module_utils overrides facts.py
|
||||
test_override:
|
||||
register: result
|
||||
|
||||
- name: Make sure the we used the local facts.py, not the one shipped with ansible
|
||||
assert:
|
||||
that:
|
||||
- 'result["data"] == "overridden facts.py"'
|
||||
|
||||
- name: Test that importing a module that only exists inside of a submodule does not work
|
||||
test_failure:
|
||||
ignore_errors: True
|
||||
register: result
|
||||
|
||||
- name: Make sure we failed in AnsiBallZ
|
||||
assert:
|
||||
that:
|
||||
- 'result["failed"] == True'
|
||||
- '"Could not find imported module support code for test_failure. Looked for either foo or zebra" == result["msg"]'
|
||||
5
test/integration/targets/module_utils/runme.sh
Executable file
5
test/integration/targets/module_utils/runme.sh
Executable file
|
|
@ -0,0 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -eux
|
||||
|
||||
ansible-playbook module_utils_test.yml -i ../../inventory -v "$@"
|
||||
|
|
@ -6,6 +6,7 @@ grep '^#!' -rIn . \
|
|||
| grep ':1:' | sed 's/:1:/:/' | grep -v -E \
|
||||
-e '^\./lib/ansible/modules/' \
|
||||
-e '^\./test/integration/targets/[^/]*/library/[^/]*:#!powershell$' \
|
||||
-e '^\./test/integration/targets/[^/]*/library/[^/]*:#!/usr/bin/python$' \
|
||||
-e '^\./test/sanity/validate-modules/validate-modules:#!/usr/bin/env python2$' \
|
||||
-e '^\./hacking/cherrypick.py:#!/usr/bin/env python3$' \
|
||||
-e ':#!/bin/sh$' \
|
||||
|
|
|
|||
134
test/units/executor/module_common/test_recursive_finder.py
Normal file
134
test/units/executor/module_common/test_recursive_finder.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com>
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import imp
|
||||
import zipfile
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from io import BytesIO, StringIO
|
||||
|
||||
import pytest
|
||||
|
||||
import ansible.errors
|
||||
from ansible.compat.six import PY2
|
||||
from ansible.compat.six.moves import builtins
|
||||
|
||||
from ansible.executor.module_common import recursive_finder
|
||||
|
||||
|
||||
original_find_module = imp.find_module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def finder_containers():
|
||||
FinderContainers = namedtuple('FinderContainers', ['py_module_names', 'py_module_cache', 'zf'])
|
||||
|
||||
py_module_names = set()
|
||||
#py_module_cache = {('__init__',): b''}
|
||||
py_module_cache = {}
|
||||
|
||||
zipoutput = BytesIO()
|
||||
zf = zipfile.ZipFile(zipoutput, mode='w', compression=zipfile.ZIP_STORED)
|
||||
#zf.writestr('ansible/__init__.py', b'')
|
||||
|
||||
return FinderContainers(py_module_names, py_module_cache, zf)
|
||||
|
||||
|
||||
def find_module_foo(module_utils_data, *args, **kwargs):
|
||||
if args[0] == 'foo':
|
||||
return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo.py', ('.py', 'r', imp.PY_SOURCE))
|
||||
return original_find_module(*args, **kwargs)
|
||||
|
||||
|
||||
def find_package_foo(module_utils_data, *args, **kwargs):
|
||||
if args[0] == 'foo':
|
||||
return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo', ('', '', imp.PKG_DIRECTORY))
|
||||
return original_find_module(*args, **kwargs)
|
||||
|
||||
|
||||
class TestRecursiveFinder(object):
|
||||
def test_no_module_utils(self, finder_containers):
|
||||
name = 'ping'
|
||||
data = b'#!/usr/bin/python\nreturn \'{\"changed\": false}\''
|
||||
recursive_finder(name, data, *finder_containers)
|
||||
assert finder_containers.py_module_names == set(())
|
||||
assert finder_containers.py_module_cache == {}
|
||||
assert frozenset(finder_containers.zf.namelist()) == frozenset()
|
||||
|
||||
def test_from_import_toplevel_package(self, finder_containers, mocker):
|
||||
if PY2:
|
||||
module_utils_data = BytesIO(b'# License\ndef do_something():\n pass\n')
|
||||
else:
|
||||
module_utils_data = StringIO(u'# License\ndef do_something():\n pass\n')
|
||||
mocker.patch('imp.find_module', side_effect=partial(find_package_foo, module_utils_data))
|
||||
mocker.patch('ansible.executor.module_common._slurp', side_effect= lambda x: b'# License\ndef do_something():\n pass\n')
|
||||
|
||||
name = 'ping'
|
||||
data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo'
|
||||
recursive_finder(name, data, *finder_containers)
|
||||
mocker.stopall()
|
||||
|
||||
assert finder_containers.py_module_names == set((('foo', '__init__'),))
|
||||
assert finder_containers.py_module_cache == {}
|
||||
assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo/__init__.py',))
|
||||
|
||||
def test_from_import_toplevel_module(self, finder_containers, mocker):
|
||||
if PY2:
|
||||
module_utils_data = BytesIO(b'# License\ndef do_something():\n pass\n')
|
||||
else:
|
||||
module_utils_data = StringIO(u'# License\ndef do_something():\n pass\n')
|
||||
mocker.patch('imp.find_module', side_effect=partial(find_module_foo, module_utils_data))
|
||||
|
||||
name = 'ping'
|
||||
data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo'
|
||||
recursive_finder(name, data, *finder_containers)
|
||||
mocker.stopall()
|
||||
|
||||
assert finder_containers.py_module_names == set((('foo',),))
|
||||
assert finder_containers.py_module_cache == {}
|
||||
assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo.py',))
|
||||
|
||||
#
|
||||
# Test importing six with many permutations because it is not a normal module
|
||||
#
|
||||
def test_from_import_six(self, finder_containers):
|
||||
name = 'ping'
|
||||
data = b'#!/usr/bin/python\nfrom ansible.module_utils import six'
|
||||
recursive_finder(name, data, *finder_containers)
|
||||
assert finder_containers.py_module_names == set((('six',),))
|
||||
assert finder_containers.py_module_cache == {}
|
||||
assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
|
||||
|
||||
def test_import_six(self, finder_containers):
|
||||
name = 'ping'
|
||||
data = b'#!/usr/bin/python\nimport ansible.module_utils.six'
|
||||
recursive_finder(name, data, *finder_containers)
|
||||
assert finder_containers.py_module_names == set((('six',),))
|
||||
assert finder_containers.py_module_cache == {}
|
||||
assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
|
||||
|
||||
def test_import_six_from_many_submodules(self, finder_containers):
|
||||
name = 'ping'
|
||||
data = b'#!/usr/bin/python\nfrom ansible.module_utils.six.moves.urllib.parse import urlparse'
|
||||
recursive_finder(name, data, *finder_containers)
|
||||
assert finder_containers.py_module_names == set((('six',),))
|
||||
assert finder_containers.py_module_cache == {}
|
||||
assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
|
||||
Loading…
Reference in a new issue