mirror of
https://github.com/ansible-collections/community.general.git
synced 2024-09-14 20:13:21 +02:00
Overhaul ansible-test import analysis. (#22888)
- Add support for module_utils packages for better organization. - Add support for "virtual" module_utils packages such as `six`.
This commit is contained in:
parent
ac992eff1c
commit
4fdeade389
2 changed files with 110 additions and 27 deletions
|
@ -130,19 +130,22 @@ class PathMapper(object):
|
||||||
:type path: str
|
:type path: str
|
||||||
:rtype: list[str]
|
:rtype: list[str]
|
||||||
"""
|
"""
|
||||||
name, ext = os.path.splitext(os.path.split(path)[1])
|
ext = os.path.splitext(os.path.split(path)[1])[1]
|
||||||
|
|
||||||
if path.startswith('lib/ansible/module_utils/'):
|
if path.startswith('lib/ansible/module_utils/'):
|
||||||
if ext == '.py':
|
if ext == '.py':
|
||||||
return self.get_python_module_utils_usage(name)
|
return self.get_python_module_utils_usage(path)
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def get_python_module_utils_usage(self, name):
|
def get_python_module_utils_usage(self, path):
|
||||||
"""
|
"""
|
||||||
:type name: str
|
:type path: str
|
||||||
:rtype: list[str]
|
:rtype: list[str]
|
||||||
"""
|
"""
|
||||||
|
if path == 'lib/ansible/module_utils/__init__.py':
|
||||||
|
return []
|
||||||
|
|
||||||
if not self.python_module_utils_imports:
|
if not self.python_module_utils_imports:
|
||||||
display.info('Analyzing python module_utils imports...')
|
display.info('Analyzing python module_utils imports...')
|
||||||
before = time.time()
|
before = time.time()
|
||||||
|
@ -150,7 +153,12 @@ class PathMapper(object):
|
||||||
after = time.time()
|
after = time.time()
|
||||||
display.info('Processed %d python module_utils in %d second(s).' % (len(self.python_module_utils_imports), after - before))
|
display.info('Processed %d python module_utils in %d second(s).' % (len(self.python_module_utils_imports), after - before))
|
||||||
|
|
||||||
return sorted(self.python_module_utils_imports.get(name, set()))
|
name = os.path.splitext(path)[0].replace('/', '.')[4:]
|
||||||
|
|
||||||
|
if name.endswith('.__init__'):
|
||||||
|
name = name[:-9]
|
||||||
|
|
||||||
|
return sorted(self.python_module_utils_imports[name])
|
||||||
|
|
||||||
def classify(self, path):
|
def classify(self, path):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -11,14 +11,20 @@ from lib.util import (
|
||||||
ApplicationError,
|
ApplicationError,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
VIRTUAL_PACKAGES = set([
|
||||||
|
'ansible.module_utils.six',
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
def get_python_module_utils_imports(compile_targets):
|
def get_python_module_utils_imports(compile_targets):
|
||||||
"""Return a dictionary of python file paths mapped to sets of module_utils names.
|
"""Return a dictionary of module_utils names mapped to sets of python file paths.
|
||||||
:type compile_targets: list[TestTarget]
|
:type compile_targets: list[TestTarget]
|
||||||
:rtype: dict[str, set[str]]
|
:rtype: dict[str, set[str]]
|
||||||
"""
|
"""
|
||||||
module_utils_files = (os.path.splitext(filename) for filename in os.listdir('lib/ansible/module_utils'))
|
|
||||||
module_utils = sorted(name[0] for name in module_utils_files if name[0] != '__init__' and name[1] == '.py')
|
module_utils = enumerate_module_utils()
|
||||||
|
virtual_utils = set(m for m in module_utils if any(m.startswith('%s.' % v) for v in VIRTUAL_PACKAGES))
|
||||||
|
module_utils -= virtual_utils
|
||||||
|
|
||||||
imports_by_target_path = {}
|
imports_by_target_path = {}
|
||||||
|
|
||||||
|
@ -39,9 +45,33 @@ def get_python_module_utils_imports(compile_targets):
|
||||||
|
|
||||||
results = set([import_name])
|
results = set([import_name])
|
||||||
|
|
||||||
import_path = os.path.join('lib/ansible/module_utils', '%s.py' % import_name)
|
# virtual packages depend on the modules they contain instead of the reverse
|
||||||
|
if import_name in VIRTUAL_PACKAGES:
|
||||||
|
for sub_import in sorted(virtual_utils):
|
||||||
|
if sub_import.startswith('%s.' % import_name):
|
||||||
|
if sub_import in seen:
|
||||||
|
continue
|
||||||
|
|
||||||
|
seen.add(sub_import)
|
||||||
|
|
||||||
|
matches = sorted(recurse_import(sub_import, depth + 1, seen))
|
||||||
|
|
||||||
|
for result in matches:
|
||||||
|
results.add(result)
|
||||||
|
|
||||||
|
import_path = os.path.join('lib/', '%s.py' % import_name.replace('.', '/'))
|
||||||
|
|
||||||
|
if import_path not in imports_by_target_path:
|
||||||
|
import_path = os.path.join('lib/', import_name.replace('.', '/'), '__init__.py')
|
||||||
|
|
||||||
|
if import_path not in imports_by_target_path:
|
||||||
|
raise ApplicationError('Cannot determine path for module_utils import: %s' % import_name)
|
||||||
|
|
||||||
|
# process imports in reverse so the deepest imports come first
|
||||||
|
for name in sorted(imports_by_target_path[import_path], reverse=True):
|
||||||
|
if name in virtual_utils:
|
||||||
|
continue
|
||||||
|
|
||||||
for name in sorted(imports_by_target_path.get(import_path, set())):
|
|
||||||
if name in seen:
|
if name in seen:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
@ -67,12 +97,18 @@ def get_python_module_utils_imports(compile_targets):
|
||||||
display.info('%s inherits import %s via %s' % (target_path, module_util_import, module_util), verbosity=6)
|
display.info('%s inherits import %s via %s' % (target_path, module_util_import, module_util), verbosity=6)
|
||||||
imports_by_target_path[target_path].add(module_util_import)
|
imports_by_target_path[target_path].add(module_util_import)
|
||||||
|
|
||||||
imports = dict([(module_util, set()) for module_util in module_utils])
|
imports = dict([(module_util, set()) for module_util in module_utils | virtual_utils])
|
||||||
|
|
||||||
for target_path in imports_by_target_path:
|
for target_path in imports_by_target_path:
|
||||||
for module_util in imports_by_target_path[target_path]:
|
for module_util in imports_by_target_path[target_path]:
|
||||||
imports[module_util].add(target_path)
|
imports[module_util].add(target_path)
|
||||||
|
|
||||||
|
# for purposes of mapping module_utils to paths, treat imports of virtual utils the same as the parent package
|
||||||
|
for virtual_util in virtual_utils:
|
||||||
|
parent_package = '.'.join(virtual_util.split('.')[:-1])
|
||||||
|
imports[virtual_util] = imports[parent_package]
|
||||||
|
display.info('%s reports imports from parent package %s' % (virtual_util, parent_package), verbosity=6)
|
||||||
|
|
||||||
for module_util in sorted(imports):
|
for module_util in sorted(imports):
|
||||||
if not len(imports[module_util]):
|
if not len(imports[module_util]):
|
||||||
display.warning('No imports found which use the "%s" module_util.' % module_util)
|
display.warning('No imports found which use the "%s" module_util.' % module_util)
|
||||||
|
@ -80,6 +116,34 @@ def get_python_module_utils_imports(compile_targets):
|
||||||
return imports
|
return imports
|
||||||
|
|
||||||
|
|
||||||
|
def enumerate_module_utils():
|
||||||
|
"""Return a list of available module_utils imports.
|
||||||
|
:rtype: set[str]
|
||||||
|
"""
|
||||||
|
module_utils = []
|
||||||
|
base_path = 'lib/ansible/module_utils'
|
||||||
|
|
||||||
|
for root, _, file_names in os.walk(base_path):
|
||||||
|
for file_name in file_names:
|
||||||
|
path = os.path.join(root, file_name)
|
||||||
|
name, ext = os.path.splitext(file_name)
|
||||||
|
|
||||||
|
if path == 'lib/ansible/module_utils/__init__.py':
|
||||||
|
continue
|
||||||
|
|
||||||
|
if ext != '.py':
|
||||||
|
continue
|
||||||
|
|
||||||
|
if name == '__init__':
|
||||||
|
module_util = root
|
||||||
|
else:
|
||||||
|
module_util = os.path.join(root, name)
|
||||||
|
|
||||||
|
module_utils.append(module_util[4:].replace('/', '.'))
|
||||||
|
|
||||||
|
return set(module_utils)
|
||||||
|
|
||||||
|
|
||||||
def extract_python_module_utils_imports(path, module_utils):
|
def extract_python_module_utils_imports(path, module_utils):
|
||||||
"""Return a list of module_utils imports found in the specified source file.
|
"""Return a list of module_utils imports found in the specified source file.
|
||||||
:type path: str
|
:type path: str
|
||||||
|
@ -110,12 +174,21 @@ class ModuleUtilFinder(ast.NodeVisitor):
|
||||||
:type path: str
|
:type path: str
|
||||||
:type module_utils: set[str]
|
:type module_utils: set[str]
|
||||||
"""
|
"""
|
||||||
super(ModuleUtilFinder, self).__init__()
|
|
||||||
|
|
||||||
self.path = path
|
self.path = path
|
||||||
self.module_utils = module_utils
|
self.module_utils = module_utils
|
||||||
self.imports = set()
|
self.imports = set()
|
||||||
|
|
||||||
|
# implicitly import parent package
|
||||||
|
|
||||||
|
if path.endswith('/__init__.py'):
|
||||||
|
path = os.path.split(path)[0]
|
||||||
|
|
||||||
|
if path.startswith('lib/ansible/module_utils/'):
|
||||||
|
package = os.path.split(path)[0].replace('/', '.')[4:]
|
||||||
|
|
||||||
|
if package != 'ansible.module_utils' and package not in VIRTUAL_PACKAGES:
|
||||||
|
self.add_import(package, 0)
|
||||||
|
|
||||||
# noinspection PyPep8Naming
|
# noinspection PyPep8Naming
|
||||||
# pylint: disable=locally-disabled, invalid-name
|
# pylint: disable=locally-disabled, invalid-name
|
||||||
def visit_Import(self, node):
|
def visit_Import(self, node):
|
||||||
|
@ -127,7 +200,7 @@ class ModuleUtilFinder(ast.NodeVisitor):
|
||||||
for alias in node.names:
|
for alias in node.names:
|
||||||
if alias.name.startswith('ansible.module_utils.'):
|
if alias.name.startswith('ansible.module_utils.'):
|
||||||
# import ansible.module_utils.MODULE[.MODULE]
|
# import ansible.module_utils.MODULE[.MODULE]
|
||||||
self.add_import(alias.name.split('.')[2], node.lineno)
|
self.add_import(alias.name, node.lineno)
|
||||||
|
|
||||||
# noinspection PyPep8Naming
|
# noinspection PyPep8Naming
|
||||||
# pylint: disable=locally-disabled, invalid-name
|
# pylint: disable=locally-disabled, invalid-name
|
||||||
|
@ -140,28 +213,30 @@ class ModuleUtilFinder(ast.NodeVisitor):
|
||||||
if not node.module:
|
if not node.module:
|
||||||
return
|
return
|
||||||
|
|
||||||
if node.module == 'ansible.module_utils':
|
if node.module == 'ansible.module_utils' or node.module.startswith('ansible.module_utils.'):
|
||||||
for alias in node.names:
|
for alias in node.names:
|
||||||
# from ansible.module_utils import MODULE[, MODULE]
|
# from ansible.module_utils import MODULE[, MODULE]
|
||||||
self.add_import(alias.name, node.lineno)
|
# from ansible.module_utils.MODULE[.MODULE] import MODULE[, MODULE]
|
||||||
elif node.module.startswith('ansible.module_utils.'):
|
self.add_import('%s.%s' % (node.module, alias.name), node.lineno)
|
||||||
# from ansible.module_utils.MODULE[.MODULE]
|
|
||||||
self.add_import(node.module.split('.')[2], node.lineno)
|
|
||||||
|
|
||||||
def add_import(self, name, line_number):
|
def add_import(self, name, line_number):
|
||||||
"""
|
"""
|
||||||
:type name: str
|
:type name: str
|
||||||
:type line_number: int
|
:type line_number: int
|
||||||
"""
|
"""
|
||||||
if name in self.imports:
|
import_name = name
|
||||||
return # duplicate imports are ignored
|
|
||||||
|
|
||||||
if name not in self.module_utils:
|
while len(name) > len('ansible.module_utils.'):
|
||||||
if self.path.startswith('test/'):
|
if name in self.module_utils:
|
||||||
return # invalid imports in tests are ignored
|
if name not in self.imports:
|
||||||
|
display.info('%s:%d imports module_utils: %s' % (self.path, line_number, name), verbosity=5)
|
||||||
|
self.imports.add(name)
|
||||||
|
|
||||||
raise Exception('%s:%d Invalid module_util import: %s' % (self.path, line_number, name))
|
return # duplicate imports are ignored
|
||||||
|
|
||||||
display.info('%s:%d imports module_utils: %s' % (self.path, line_number, name), verbosity=5)
|
name = '.'.join(name.split('.')[:-1])
|
||||||
|
|
||||||
self.imports.add(name)
|
if self.path.startswith('test/'):
|
||||||
|
return # invalid imports in tests are ignored
|
||||||
|
|
||||||
|
raise ApplicationError('%s:%d Invalid module_utils import: %s' % (self.path, line_number, import_name))
|
||||||
|
|
Loading…
Reference in a new issue