Merge pull request #11643 from bcoca/meta_inventory_and_fixes

Meta inventory and fixes

CHANGELOG.md

@@ -19,6 +19,8 @@ Major Changes:
  * template code now retains types for bools and numbers instead of turning them into strings.
    If you need the old behaviour, quote the value and it will get passed around as a string
  * Consolidated code from modules using urllib2 to normalize features, TLS and SNI support
+ * Consiidated code from modules using urllib2 to normalize features, TLS and SNI support
+ * added meta: refresh_inventory to force rereading the inventory in a play
 
 Deprecated Modules (new ones in parens):
   * ec2_ami_search (ec2_ami_find)

lib/ansible/constants.py

@@ -143,6 +143,7 @@ DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECU
 DEFAULT_GATHERING         = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
 DEFAULT_LOG_PATH          = shell_expand_path(get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', ''))
 DEFAULT_FORCE_HANDLERS    = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True)
+DEFAULT_INVENTORY_IGNORE  = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True)
 
 # selinux
 DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf, ramfs', islist=True)
@@ -197,7 +198,7 @@ HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  '
 SYSTEM_WARNINGS                = get_config(p, DEFAULTS, 'system_warnings', 'ANSIBLE_SYSTEM_WARNINGS', True, boolean=True)
 DEPRECATION_WARNINGS           = get_config(p, DEFAULTS, 'deprecation_warnings', 'ANSIBLE_DEPRECATION_WARNINGS', True, boolean=True)
 DEFAULT_CALLABLE_WHITELIST     = get_config(p, DEFAULTS, 'callable_whitelist', 'ANSIBLE_CALLABLE_WHITELIST', [], islist=True)
-COMMAND_WARNINGS               = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', False, boolean=True)
+COMMAND_WARNINGS               = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', True, boolean=True)
 DEFAULT_LOAD_CALLBACK_PLUGINS  = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True)
 DEFAULT_CALLBACK_WHITELIST     = get_config(p, DEFAULTS, 'callback_whitelist', 'ANSIBLE_CALLBACK_WHITELIST', [], islist=True)
 RETRY_FILES_ENABLED            = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True)

lib/ansible/inventory/__init__.py

@@ -26,15 +26,12 @@ import re
 import stat
 
 from ansible import constants as C
-from ansible import errors
+from ansible.errors import AnsibleError
 
-from ansible.inventory.ini import InventoryParser
+from ansible.inventory.dir import InventoryDirectory, get_file_parser
-from ansible.inventory.script import InventoryScript
-from ansible.inventory.dir import InventoryDirectory
 from ansible.inventory.group import Group
 from ansible.inventory.host import Host
 from ansible.plugins import vars_loader
-from ansible.utils.path import is_executable
 from ansible.utils.vars import combine_vars
 
 class Inventory(object):
@@ -63,6 +60,7 @@ class Inventory(object):
         self._hosts_cache    = {}
         self._groups_list    = {}
         self._pattern_cache  = {}
+        self._vars_plugins   = []
 
         # to be set by calling set_playbook_basedir by playbook code
         self._playbook_basedir = None
@@ -75,6 +73,10 @@ class Inventory(object):
         self._also_restriction = None
         self._subset = None
 
+        self.parse_inventory(host_list)
+
+    def parse_inventory(self, host_list):
+
         if isinstance(host_list, basestring):
             if "," in host_list:
                 host_list = host_list.split(",")
@@ -102,51 +104,22 @@ class Inventory(object):
                     else:
                         all.add_host(Host(x))
         elif os.path.exists(host_list):
+            #TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins'
             if os.path.isdir(host_list):
                 # Ensure basedir is inside the directory
-                self.host_list = os.path.join(self.host_list, "")
+                host_list = os.path.join(self.host_list, "")
                 self.parser = InventoryDirectory(loader=self._loader, filename=host_list)
+            else:
+                self.parser = get_file_parser(hostsfile, self._loader)
+                vars_loader.add_directory(self.basedir(), with_subdir=True)
+
+            if self.parser:
                 self.groups = self.parser.groups.values()
             else:
-                # check to see if the specified file starts with a
+                # should never happen, but JIC
-                # shebang (#!/), so if an error is raised by the parser
+                raise AnsibleError("Unable to parse %s as an inventory source" % host_list)
-                # class we can show a more apropos error
-                shebang_present = False
-                try:
-                    with open(host_list, "r") as inv_file:
-                        first_line = inv_file.readline()
-                        if first_line.startswith("#!"):
-                            shebang_present = True
-                except IOError:
-                    pass
 
-                if is_executable(host_list):
+            self._vars_plugins = [ x for x in vars_loader.all(self) ]
-                    try:
-                        self.parser = InventoryScript(loader=self._loader, filename=host_list)
-                        self.groups = self.parser.groups.values()
-                    except errors.AnsibleError:
-                        if not shebang_present:
-                            raise errors.AnsibleError("The file %s is marked as executable, but failed to execute correctly. " % host_list + \
-                                                      "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % host_list)
-                        else:
-                            raise
-                else:
-                    try:
-                        self.parser = InventoryParser(filename=host_list)
-                        self.groups = self.parser.groups.values()
-                    except errors.AnsibleError:
-                        if shebang_present:
-                            raise errors.AnsibleError("The file %s looks like it should be an executable inventory script, but is not marked executable. " % host_list + \
-                                                      "Perhaps you want to correct this with `chmod +x %s`?" % host_list)
-                        else:
-                            raise
-
-            vars_loader.add_directory(self.basedir(), with_subdir=True)
-        else:
-            raise errors.AnsibleError("Unable to find an inventory file (%s), "
-                                      "specify one with -i ?" % host_list)
-
-        self._vars_plugins = [ x for x in vars_loader.all(self) ]
 
         # FIXME: shouldn't be required, since the group/host vars file
         #        management will be done in VariableManager
@@ -166,7 +139,7 @@ class Inventory(object):
             else:
                 return fnmatch.fnmatch(str, pattern_str)
         except Exception, e:
-            raise errors.AnsibleError('invalid host pattern: %s' % pattern_str)
+            raise AnsibleError('invalid host pattern: %s' % pattern_str)
 
     def _match_list(self, items, item_attr, pattern_str):
         results = []
@@ -176,7 +149,7 @@ class Inventory(object):
             else:
                 pattern = re.compile(pattern_str[1:])
         except Exception, e:
-            raise errors.AnsibleError('invalid host pattern: %s' % pattern_str)
+            raise AnsibleError('invalid host pattern: %s' % pattern_str)
 
         for item in items:
             if pattern.match(getattr(item, item_attr)):
@@ -286,7 +259,7 @@ class Inventory(object):
             first = int(first)
             if last:
                 if first < 0:
-                    raise errors.AnsibleError("invalid range: negative indices cannot be used as the first item in a range")
+                    raise AnsibleError("invalid range: negative indices cannot be used as the first item in a range")
                 last = int(last)
             else:
                 last = first
@@ -324,7 +297,7 @@ class Inventory(object):
             else:
                 return [ hosts[left] ]
         except IndexError:
-            raise errors.AnsibleError("no hosts matching the pattern '%s' were found" % pat)
+            raise AnsibleError("no hosts matching the pattern '%s' were found" % pat)
 
     def _create_implicit_localhost(self, pattern):
         new_host = Host(pattern)
@@ -467,7 +440,7 @@ class Inventory(object):
 
         host = self.get_host(hostname)
         if host is None:
-            raise errors.AnsibleError("host not found: %s" % hostname)
+            raise AnsibleError("host not found: %s" % hostname)
 
         vars = {}
 
@@ -499,7 +472,7 @@ class Inventory(object):
             self.groups.append(group)
             self._groups_list = None  # invalidate internal cache 
         else:
-            raise errors.AnsibleError("group already in inventory: %s" % group.name)
+            raise AnsibleError("group already in inventory: %s" % group.name)
 
     def list_hosts(self, pattern="all"):
 
@@ -670,3 +643,14 @@ class Inventory(object):
         # all done, results is a dictionary of variables for this particular host.
         return results
 
+    def refresh_inventory(self):
+
+        self.clear_pattern_cache()
+
+        self._hosts_cache    = {}
+        self._vars_per_host  = {}
+        self._vars_per_group = {}
+        self._groups_list    = {}
+        self.groups = []
+
+        self.parse_inventory(self.host_list)

lib/ansible/inventory/dir.py

@@ -27,11 +27,58 @@ from ansible.errors import AnsibleError
 
 from ansible.inventory.host import Host
 from ansible.inventory.group import Group
-from ansible.inventory.ini import InventoryParser
-from ansible.inventory.script import InventoryScript
-from ansible.utils.path import is_executable
 from ansible.utils.vars import combine_vars
 
+from ansible.utils.path import is_executable
+from ansible.inventory.ini import InventoryParser as InventoryINIParser
+from ansible.inventory.script import InventoryScript
+
+__all__ = ['get_file_parser']
+
+def get_file_parser(hostsfile, loader):
+    # check to see if the specified file starts with a
+    # shebang (#!/), so if an error is raised by the parser
+    # class we can show a more apropos error
+
+    shebang_present = False
+    processed = False
+    myerr = []
+    parser = None
+
+    try:
+        inv_file = open(hostsfile)
+        first_line = inv_file.readlines()[0]
+        inv_file.close()
+        if first_line.startswith('#!'):
+            shebang_present = True
+    except:
+        pass
+
+    if is_executable(hostsfile):
+        try:
+            parser = InventoryScript(loader=loader, filename=hostsfile)
+            processed = True
+        except Exception as e:
+            myerr.append("The file %s is marked as executable, but failed to execute correctly. " % hostsfile + \
+                            "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % hostsfile)
+            myerr.append(str(e))
+
+    if not processed:
+        try:
+            parser = InventoryINIParser(filename=hostsfile)
+            processed = True
+        except Exception as e:
+            if shebang_present and not is_executable(hostsfile):
+                myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \
+                              "Perhaps you want to correct this with `chmod +x %s`?" % hostsfile)
+            else:
+                myerr.append(str(e))
+
+    if not processed and myerr:
+        raise AnsibleError( '\n'.join(myerr) )
+
+    return parser
+
 class InventoryDirectory(object):
     ''' Host inventory parser for ansible using a directory of inventories. '''
 
@@ -48,7 +95,7 @@ class InventoryDirectory(object):
         for i in self.names:
 
             # Skip files that end with certain extensions or characters
-            if any(i.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo")):
+            if any(i.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
                 continue
             # Skip hidden files
             if i.startswith('.') and not i.startswith('./'):
@@ -59,10 +106,14 @@ class InventoryDirectory(object):
             fullpath = os.path.join(self.directory, i)
             if os.path.isdir(fullpath):
                 parser = InventoryDirectory(loader=loader, filename=fullpath)
-            elif is_executable(fullpath):
-                parser = InventoryScript(loader=loader, filename=fullpath)
             else:
-                parser = InventoryParser(filename=fullpath)
+                parser = get_file_parser(fullpath, loader)
+                if parser is None:
+                    #FIXME: needs to use display
+                    import warnings
+                    warnings.warning("Could not find parser for %s, skipping" % fullpath)
+                    continue
+
             self.parsers.append(parser)
 
             # retrieve all groups and hosts form the parser and add them to

lib/ansible/plugins/inventory/README.md

@@ -0,0 +1 @@
+These are not currently in use, but this is what the future of inventory will become after 2.0

lib/ansible/plugins/inventory/directory.py

@@ -22,11 +22,14 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
+from ansible import constants as C
 
 from . aggregate import InventoryAggregateParser
 
 class InventoryDirectoryParser(InventoryAggregateParser):
 
+    CONDITION="is_dir(%s)"
+
     def __init__(self, inven_directory):
         directory = inven_directory
         names = os.listdir(inven_directory)
@@ -35,7 +38,7 @@ class InventoryDirectoryParser(InventoryAggregateParser):
         # Clean up the list of filenames
         for filename in names:
             # Skip files that end with certain extensions or characters
-            if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")):
+            if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
                 continue
             # Skip hidden files
             if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)):

lib/ansible/plugins/inventory/ini.py

@@ -23,10 +23,13 @@ __metaclass__ = type
 
 import os
 
+from ansible import constants as C
 from . import InventoryParser
 
 class InventoryIniParser(InventoryAggregateParser):
 
+    CONDITION="is_file(%s)"
+
     def __init__(self, inven_directory):
         directory = inven_directory
         names = os.listdir(inven_directory)
@@ -35,7 +38,7 @@ class InventoryIniParser(InventoryAggregateParser):
         # Clean up the list of filenames
         for filename in names:
             # Skip files that end with certain extensions or characters
-            if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")):
+            if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
                 continue
             # Skip hidden files
             if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)):

lib/ansible/plugins/inventory/script.py

@@ -0,0 +1,31 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+#############################################
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+
+from ansible import constants as C
+from . import InventoryParser
+
+class InventoryScriptParser(InventoryParser):
+
+    CONDITION="is_file(%s) and is_executable(%s)"

lib/ansible/plugins/strategies/__init__.py

@@ -506,3 +506,21 @@ class StrategyBase:
         self._display.banner(msg)
 
         return ret
+
+    def _execute_meta(self, task, play_context, iterator):
+
+        # meta tasks store their args in the _raw_params field of args,
+        # since they do not use k=v pairs, so get that
+        meta_action = task.args.get('_raw_params')
+
+        if meta_action == 'noop':
+            # FIXME: issue a callback for the noop here?
+            pass
+        elif meta_action == 'flush_handlers':
+            self.run_handlers(iterator, play_context)
+        elif meta_action == 'refresh_inventory':
+            self._inventory.refresh_inventory()
+        #elif meta_action == 'reset_connection':
+        #    connection_info.connection.close()
+        else:
+            raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)

lib/ansible/plugins/strategies/linear.py

@@ -178,16 +178,7 @@ class StrategyModule(StrategyBase):
                             continue
 
                     if task.action == 'meta':
-                        # meta tasks store their args in the _raw_params field of args,
+                        self._execute_meta(task, play_context, iterator)
-                        # since they do not use k=v pairs, so get that
-                        meta_action = task.args.get('_raw_params')
-                        if meta_action == 'noop':
-                            # FIXME: issue a callback for the noop here?
-                            continue
-                        elif meta_action == 'flush_handlers':
-                            self.run_handlers(iterator, play_context)
-                        else:
-                            raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)
                     else:
                         # handle step if needed, skip meta actions as they are used internally
                         if self._step and choose_step: