From 960388143f086bd229b3d5f779b7b9e2b78c8bd7 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 23 Jan 2019 12:06:54 -0500 Subject: [PATCH] faster config loading (#48333) * faster config loading - used already loaded module var instead of doc functions - add preload to populate config defs cache - avoid debug work when not in debug - generators, force consumption --- lib/ansible/executor/playbook_executor.py | 5 ++++ lib/ansible/plugins/loader.py | 33 +++++++++++------------ lib/ansible/utils/plugin_docs.py | 2 +- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/ansible/executor/playbook_executor.py b/lib/ansible/executor/playbook_executor.py index 99be3e3e23..c2ac125bd3 100644 --- a/lib/ansible/executor/playbook_executor.py +++ b/lib/ansible/executor/playbook_executor.py @@ -27,6 +27,7 @@ from ansible.executor.task_queue_manager import TaskQueueManager from ansible.module_utils._text import to_native, to_text from ansible.playbook import Playbook from ansible.template import Templar +from ansible.plugins.loader import connection_loader, shell_loader from ansible.utils.helpers import pct_to_int from ansible.module_utils.parsing.convert_bool import boolean from ansible.utils.path import makedirs_safe @@ -76,6 +77,10 @@ class PlaybookExecutor: entrylist = [] entry = {} try: + # preload become/connecition/shell to set config defs cached + list(connection_loader.all(class_only=True)) + list(shell_loader.all(class_only=True)) + for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) # FIXME: move out of inventory self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path))) diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index 4e45e98076..c6c32c8e5d 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -20,9 +20,10 @@ from ansible import constants as C from ansible.errors import AnsibleError from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.parsing.utils.yaml import from_yaml +from ansible.parsing.yaml.loader import AnsibleLoader from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE from ansible.utils.display import Display -from ansible.utils.plugin_docs import get_docstring +from ansible.utils.plugin_docs import add_fragments display = Display() @@ -199,7 +200,7 @@ class PluginLoader: self._paths = reordered_paths return reordered_paths - def _load_config_defs(self, name, path): + def _load_config_defs(self, name, module, path): ''' Reads plugin docs to find configuration setting definitions, to push to config manager for later use ''' # plugins w/o class name don't support config @@ -208,7 +209,9 @@ class PluginLoader: # if type name != 'module_doc_fragment': if type_name in C.CONFIGURABLE_PLUGINS: - dstring = get_docstring(path, fragment_loader, verbose=False, ignore_errors=True)[0] + dstring = AnsibleLoader(getattr(module, 'DOCUMENTATION', ''), file_name=path).get_single_data() + if dstring: + add_fragments(dstring, path, fragment_loader=fragment_loader) if dstring and 'options' in dstring and isinstance(dstring['options'], dict): C.config.initialize_plugin_configuration_definitions(type_name, name, dstring['options']) @@ -374,6 +377,7 @@ class PluginLoader: if path not in self._module_cache: self._module_cache[path] = self._load_module_source(name, path) + self._load_config_defs(name, self._module_cache[path], path) found_in_cache = False obj = getattr(self._module_cache[path], self.class_name) @@ -400,23 +404,21 @@ class PluginLoader: return None raise - # load plugin config data - if not found_in_cache: - self._load_config_defs(name, path) - self._update_object(obj, name, path) return obj def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): - msg = 'Loading %s \'%s\' from %s' % (class_name, os.path.basename(name), path) + ''' formats data to display debug info for plugin loading, also avoids processing unless really needed ''' + if C.DEFAULT_DEBUG: + msg = 'Loading %s \'%s\' from %s' % (class_name, os.path.basename(name), path) - if len(searched_paths) > 1: - msg = '%s (searched paths: %s)' % (msg, self.format_paths(searched_paths)) + if len(searched_paths) > 1: + msg = '%s (searched paths: %s)' % (msg, self.format_paths(searched_paths)) - if found_in_cache or class_only: - msg = '%s (found_in_cache=%s, class_only=%s)' % (msg, found_in_cache, class_only) + if found_in_cache or class_only: + msg = '%s (found_in_cache=%s, class_only=%s)' % (msg, found_in_cache, class_only) - display.debug(msg) + display.debug(msg) def all(self, *args, **kwargs): ''' @@ -486,6 +488,7 @@ class PluginLoader: if path not in self._module_cache: try: module = self._load_module_source(name, path) + self._load_config_defs(basename, module, path) except Exception as e: display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_text(e))) self._module_cache[path] = module @@ -516,10 +519,6 @@ class PluginLoader: except TypeError as e: display.warning("Skipping plugin (%s) as it seems to be incomplete: %s" % (path, to_text(e))) - # load plugin config data - if not found_in_cache: - self._load_config_defs(basename, path) - self._update_object(obj, basename, path) yield obj diff --git a/lib/ansible/utils/plugin_docs.py b/lib/ansible/utils/plugin_docs.py index 6bf9d2a801..6cbc50859d 100644 --- a/lib/ansible/utils/plugin_docs.py +++ b/lib/ansible/utils/plugin_docs.py @@ -8,7 +8,7 @@ from ansible.errors import AnsibleError, AnsibleAssertionError from ansible.module_utils.six import string_types from ansible.module_utils._text import to_native from ansible.module_utils.common._collections_compat import MutableMapping, MutableSet, MutableSequence -from ansible.parsing.plugin_docs import read_docstring, read_docstub +from ansible.parsing.plugin_docs import read_docstring from ansible.parsing.yaml.loader import AnsibleLoader from ansible.utils.display import Display