add inventory caching & use in virtualbox inventory plugin (#34510)

* Inventory caching

* Add inventory caching for virtualbox

* Don't populate cache for virtualbox with stdout, use a dict of inventory instead

* Fix error creating the cache dir if it doesn't exist

* Keep cache default False and set to True in VariableManager __init__

* Check all groups before determining if a host is ungrouped.

lib/ansible/inventory/manager.py

@@ -142,7 +142,7 @@ class InventoryManager(object):
             self._sources = sources
 
         # get to work!
-        self.parse_sources()
+        self.parse_sources(cache=True)
 
     @property
     def localhost(self):
@@ -234,7 +234,7 @@ class InventoryManager(object):
 
                 # recursively deal with directory entries
                 fullpath = os.path.join(b_source, i)
-                parsed_this_one = self.parse_source(to_native(fullpath))
+                parsed_this_one = self.parse_source(to_native(fullpath), cache=cache)
                 display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one))
                 if not parsed:
                     parsed = parsed_this_one

lib/ansible/plugins/cache/__init__.py

@@ -79,12 +79,24 @@ class BaseFileCacheModule(BaseCacheModule):
         self.plugin_name = self.__module__.split('.')[-1]
         self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
         self._cache = {}
-        self._cache_dir = None
+        self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
+        self._set_inventory_cache_override(**kwargs)
+        self.validate_cache_connection()
 
-        if C.CACHE_PLUGIN_CONNECTION:
-            # expects a dir path
-            self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION))
+    def _get_cache_connection(self, source):
+        if source:
+            try:
+                return os.path.expanduser(os.path.expandvars(source))
+            except TypeError:
+                pass
 
+    def _set_inventory_cache_override(self, **kwargs):
+        if kwargs.get('cache_timeout'):
+            self._timeout = kwargs.get('cache_timeout')
+        if kwargs.get('cache_connection'):
+            self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection'))
+
+    def validate_cache_connection(self):
         if not self._cache_dir:
             raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
                                "to be set (to a writeable directory path)" % self.plugin_name)
@@ -141,7 +153,7 @@ class BaseFileCacheModule(BaseCacheModule):
     def has_expired(self, key):
 
         if self._timeout == 0:
-            return False
+            return True
 
         cachefile = "%s/%s" % (self._cache_dir, key)
         try:
@@ -284,3 +296,52 @@ class FactCache(MutableMapping):
         host_cache = self._plugin.get(key)
         host_cache.update(value)
         self._plugin.set(key, host_cache)
+
+
+class InventoryFileCacheModule(BaseFileCacheModule):
+    """
+    A caching module backed by file based storage.
+    """
+    def __init__(self, plugin_name, timeout, cache_dir):
+
+        self.plugin_name = plugin_name
+        self._timeout = timeout
+        self._cache = {}
+        self._cache_dir = self._get_cache_connection(cache_dir)
+        self.validate_cache_connection()
+        self._plugin = self.get_plugin(plugin_name)
+
+    def validate_cache_connection(self):
+        try:
+            super(InventoryFileCacheModule, self).validate_cache_connection()
+        except AnsibleError as e:
+            cache_connection_set = False
+        else:
+            cache_connection_set = True
+
+        if not cache_connection_set:
+            raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n"
+                               "ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n"
+                               "Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION."
+                               "to be set to a writeable directory path" % self.plugin_name)
+
+    def get(self, cache_key):
+
+        if not self.contains(cache_key):
+            # Check if cache file exists
+            raise KeyError
+
+        return super(InventoryFileCacheModule, self).get(cache_key)
+
+    def get_plugin(self, plugin_name):
+        plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout)
+        if not plugin:
+            raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name))
+        self._cache = {}
+        return plugin
+
+    def _load(self, path):
+        return self._plugin._load(path)
+
+    def _dump(self, value, path):
+        return self._plugin._dump(value, path)

lib/ansible/plugins/inventory/__init__.py

@@ -28,6 +28,7 @@ from collections import MutableMapping
 
 from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
 from ansible.plugins import AnsiblePlugin
+from ansible.plugins.cache import InventoryFileCacheModule
 from ansible.module_utils._text import to_bytes, to_native
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import string_types
@@ -142,6 +143,7 @@ class BaseInventoryPlugin(AnsiblePlugin):
         self._options = {}
         self.inventory = None
         self.display = display
+        self.cache = None
 
     def parse(self, inventory, loader, path, cache=True):
         ''' Populates self.groups from the given data. Raises an error on any parse failure.  '''
@@ -185,9 +187,16 @@ class BaseInventoryPlugin(AnsiblePlugin):
             raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
 
         self.set_options(direct=config)
+        if self._options.get('cache'):
+            self._set_cache_options(self._options)
 
         return config
 
+    def _set_cache_options(self, options):
+        self.cache = InventoryFileCacheModule(plugin_name=options.get('cache_plugin'),
+                                              timeout=options.get('cache_timeout'),
+                                              cache_dir=options.get('cache_connection'))
+
     def _consume_options(self, data):
         ''' update existing options from file data'''
 
@@ -213,6 +222,9 @@ class Cacheable(object):
 
     _cache = {}
 
+    def get_cache_key(self, path):
+        return "{0}_{1}_{2}".format(self.NAME, self._get_cache_prefix(path), self._get_config_identifier(path))
+
     def _get_cache_prefix(self, path):
         ''' create predictable unique prefix for plugin/inventory '''
 
@@ -226,6 +238,11 @@ class Cacheable(object):
 
         return 's_'.join([d1[:5], d2[:5]])
 
+    def _get_config_identifier(self, path):
+        ''' create predictable config-specific prefix for plugin/inventory '''
+
+        return hashlib.md5(path.encode()).hexdigest()
+
     def clear_cache(self):
         self._cache = {}
 

lib/ansible/plugins/inventory/virtualbox.py

@@ -14,6 +14,7 @@ DOCUMENTATION = '''
         - The inventory_hostname is always the 'Name' of the virtualbox instance.
     extends_documentation_fragment:
       - constructed
+      - inventory_cache
     options:
         running_only:
             description: toggles showing all vms vs only those currently running
@@ -91,7 +92,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             # constructed groups based on conditionals
             self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host)
 
-    def _populate_from_source(self, source_data):
+    def _populate_from_cache(self, source_data):
+        hostvars = source_data.pop('_meta', {}).get('hostvars', {})
+        for group in source_data:
+            if group == 'all':
+                continue
+            else:
+                self.inventory.add_group(group)
+                hosts = source_data[group].get('hosts', [])
+                for host in hosts:
+                    self._populate_host_vars([host], hostvars.get(host, {}), group)
+                self.inventory.add_child('all', group)
+        if not source_data:
+            for host in hostvars:
+                self.inventory.add_host(host)
+                self._populate_host_vars([host], hostvars.get(host, {}))
+
+    def _populate_from_source(self, source_data, using_current_cache=False):
+        if using_current_cache:
+            self._populate_from_cache(source_data)
+            return source_data
+
+        cacheable_results = {'_meta': {'hostvars': {}}}
+
         hostvars = {}
         prevkey = pref_k = ''
         current_host = None
@@ -100,6 +123,9 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
         netinfo = self.get_option('network_info_path')
 
         for line in source_data:
+            line = to_text(line)
+            if ':' not in line:
+                continue
             try:
                 k, v = line.split(':', 1)
             except:
@@ -127,8 +153,11 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             elif k == 'Groups':
                 for group in v.split('/'):
                     if group:
+                        if group not in cacheable_results:
+                            cacheable_results[group] = {'hosts': []}
                         self.inventory.add_group(group)
                         self.inventory.add_child(group, current_host)
+                        cacheable_results[group]['hosts'].append(current_host)
                 continue
 
             else:
@@ -141,10 +170,32 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
                 else:
                     if v != '':
                         hostvars[current_host][pref_k] = v
+                if self._ungrouped_host(current_host, cacheable_results):
+                    if 'ungrouped' not in cacheable_results:
+                        cacheable_results['ungrouped'] = {'hosts': []}
+                    cacheable_results['ungrouped']['hosts'].append(current_host)
 
                 prevkey = pref_k
 
         self._set_variables(hostvars)
+        for host in hostvars:
+            h = self.inventory.get_host(host)
+            cacheable_results['_meta']['hostvars'][h.name] = h.vars
+
+        return cacheable_results
+
+    def _ungrouped_host(self, host, inventory):
+        def find_host(host, inventory):
+            for k, v in inventory.items():
+                if k == '_meta':
+                    continue
+                if isinstance(v, dict):
+                    yield self._ungrouped_host(host, v)
+                elif isinstance(v, list):
+                    yield host not in v
+            yield True
+
+        return all([found_host for found_host in find_host(host, inventory)])
 
     def verify_file(self, path):
 
@@ -158,7 +209,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
 
         super(InventoryModule, self).parse(inventory, loader, path)
 
-        cache_key = self._get_cache_prefix(path)
+        cache_key = self.get_cache_key(path)
 
         config_data = self._read_config_data(path)
 
@@ -166,11 +217,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
         self._consume_options(config_data)
 
         source_data = None
-        if cache and cache_key in self._cache:
+        if cache:
+            cache = self._options.get('cache')
+
+        update_cache = False
+        if cache:
             try:
-                source_data = self._cache[cache_key]
+                source_data = self.cache.get(cache_key)
             except KeyError:
-                pass
+                update_cache = True
 
         if not source_data:
             b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict')
@@ -192,7 +247,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             except Exception as e:
                 AnsibleParserError(to_native(e))
 
-            source_data = p.stdout.read()
-            self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict')
+            source_data = p.stdout.read().splitlines()
 
-        self._populate_from_source(source_data.splitlines())
+        using_current_cache = cache and not update_cache
+        cacheable_results = self._populate_from_source(source_data, using_current_cache)
+
+        if update_cache:
+            self.cache.set(cache_key, cacheable_results)

lib/ansible/plugins/strategy/__init__.py

@@ -179,6 +179,7 @@ class StrategyBase:
         self._final_q = tqm._final_q
         self._step = getattr(tqm._options, 'step', False)
         self._diff = getattr(tqm._options, 'diff', False)
+        self.flush_cache = getattr(tqm._options, 'flush_cache', False)
 
         # the task cache is a dictionary of tuples of (host.name, task._uuid)
         # used to find the original task object of in-flight tasks and to store
@@ -956,7 +957,7 @@ class StrategyBase:
         elif meta_action == 'flush_handlers':
             self.run_handlers(iterator, play_context)
             msg = "ran handlers"
-        elif meta_action == 'refresh_inventory':
+        elif meta_action == 'refresh_inventory' or self.flush_cache:
             self._inventory.refresh_inventory()
             msg = "inventory successfully refreshed"
         elif meta_action == 'clear_facts':