Merge remote-tracking branch 'svg/varsplugins_refactor' into svg_and_inventory_refactor

bin/ansible

@@ -136,7 +136,7 @@ class Cli(object):
             if not options.ask_vault_pass:
                 vault_pass = tmp_vault_pass
 
-        inventory_manager = inventory.Inventory(options.inventory)
+        inventory_manager = inventory.Inventory(options.inventory, vault_password=vault_pass)
         if options.subset:
             inventory_manager.subset(options.subset)
         hosts = inventory_manager.list_hosts(pattern)

bin/ansible-playbook

@@ -100,11 +100,6 @@ def main(args):
     if (options.ask_vault_pass and options.vault_password_file):
             parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive")
 
-    inventory = ansible.inventory.Inventory(options.inventory)
-    inventory.subset(options.subset)
-    if len(inventory.list_hosts()) == 0:
-        raise errors.AnsibleError("provided hosts list is empty")
-
     sshpass = None
     sudopass = None
     su_pass = None
@@ -160,12 +155,14 @@ def main(args):
         if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
             raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook)
 
+    inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass)
+    inventory.subset(options.subset)
+    if len(inventory.list_hosts()) == 0:
+        raise errors.AnsibleError("provided hosts list is empty")
+
     # run all playbooks specified on the command line
     for playbook in args:
 
-        # let inventory know which playbooks are using so it can know the basedirs
-        inventory.set_playbook_basedir(os.path.dirname(playbook))
-
         stats = callbacks.AggregateStats()
         playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
         if options.step:

lib/ansible/inventory/__init__.py

@@ -38,13 +38,14 @@ class Inventory(object):
 
     __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', 
                   'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
-                  '_pattern_cache', '_vars_plugins', '_playbook_basedir']
+                  '_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir']
 
-    def __init__(self, host_list=C.DEFAULT_HOST_LIST):
+    def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None):
 
         # the host file file, or script path, or list of hosts
         # if a list, inventory data will NOT be loaded
         self.host_list = host_list
+        self._vault_password=vault_password
 
         # caching to avoid repeated calculations, particularly with
         # external inventory scripts.
@@ -55,7 +56,7 @@ class Inventory(object):
         self._groups_list    = {} 
         self._pattern_cache  = {}
 
-        # to be set by calling set_playbook_basedir by ansible-playbook
+        # to be set by calling set_playbook_basedir by playbook code
         self._playbook_basedir = None
 
         # the inventory object holds a list of groups
@@ -139,6 +140,14 @@ class Inventory(object):
 
         self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ]
 
+        # get group vars from group_vars/ files and vars plugins
+        for group in self.groups:
+            group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password))
+
+        # get host vars from host_vars/ files and vars plugins
+        for host in self.get_hosts():
+            host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password))
+
 
     def _match(self, str, pattern_str):
         if pattern_str.startswith('~'):
@@ -392,19 +401,35 @@ class Inventory(object):
                 return group
         return None
 
-    def get_group_variables(self, groupname):
+    def get_group_variables(self, groupname, update_cached=False, vault_password=None):
-        if groupname not in self._vars_per_group:
+        if groupname not in self._vars_per_group or update_cached:
-            self._vars_per_group[groupname] = self._get_group_variables(groupname)
+            self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password)
         return self._vars_per_group[groupname]
 
-    def _get_group_variables(self, groupname):
+    def _get_group_variables(self, groupname, vault_password=None):
+
         group = self.get_group(groupname)
         if group is None:
             raise Exception("group not found: %s" % groupname)
-        return group.get_variables()
 
-    def get_variables(self, hostname, vault_password=None):
+        vars = {}
-        if hostname not in self._vars_per_host:
+
+        # plugin.get_group_vars retrieves just vars for specific group
+        vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')]
+        for updated in vars_results:
+            if updated is not None:
+                vars = utils.combine_vars(vars, updated)
+
+        # get group variables set by Inventory Parsers
+        vars = utils.combine_vars(vars, group.get_variables())
+
+        # Read group_vars/ files
+        vars = utils.combine_vars(vars, self.get_group_vars(group))
+
+        return vars
+
+    def get_variables(self, hostname, update_cached=False, vault_password=None):
+        if hostname not in self._vars_per_host or update_cached:
             self._vars_per_host[hostname] = self._get_variables(hostname, vault_password=vault_password)
         return self._vars_per_host[hostname]
 
@@ -415,14 +440,31 @@ class Inventory(object):
             raise errors.AnsibleError("host not found: %s" % hostname)
 
         vars = {}
-        vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins ] 
+
+        # plugin.run retrieves all vars (also from groups) for host
+        vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')]
         for updated in vars_results:
             if updated is not None:
                 vars = utils.combine_vars(vars, updated)
 
+        # plugin.get_host_vars retrieves just vars for specific host
+        vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')]
+        for updated in vars_results:
+            if updated is not None:
+                vars = utils.combine_vars(vars, updated)
+
+        # get host variables set by Inventory Parsers
         vars = utils.combine_vars(vars, host.get_variables())
+
+        # still need to check InventoryParser per host vars
+        # which actually means InventoryScript per host,
+        # which is not performant
         if self.parser is not None:
             vars = utils.combine_vars(vars, self.parser.get_host_variables(host))
+
+        # Read host_vars/ files
+        vars = utils.combine_vars(vars, self.get_host_vars(host))
+
         return vars
 
     def add_group(self, group):
@@ -526,9 +568,72 @@ class Inventory(object):
 
     def set_playbook_basedir(self, dir):
         """
-        sets the base directory of the playbook so inventory plugins can use it to find
+        sets the base directory of the playbook so inventory can use it as a
-        variable files and other things. 
+        basedir for host_ and group_vars, and other things.
         """
-        self._playbook_basedir = dir
+        # Only update things if dir is a different playbook basedir
+        if dir != self._playbook_basedir:
+            self._playbook_basedir = dir
+            # get group vars from group_vars/ files
+            for group in self.groups:
+                group.vars = utils.combine_vars(group.vars, self.get_group_vars(group, new_pb_basedir=True))
+            # get host vars from host_vars/ files
+            for host in self.get_hosts():
+                host.vars = utils.combine_vars(host.vars, self.get_host_vars(host, new_pb_basedir=True))
 
+    def get_host_vars(self, host, new_pb_basedir=False):
+        """ Read host_vars/ files """
+        return self._get_hostgroup_vars(host=host, group=None, new_pb_basedir=False)
+
+    def get_group_vars(self, group, new_pb_basedir=False):
+        """ Read group_vars/ files """
+        return self._get_hostgroup_vars(host=None, group=group, new_pb_basedir=False)
+
+    def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False):
+        """
+        Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel
+        to the inventory base directory or in the same directory as the playbook.  Variables in the playbook
+        dir will win over the inventory dir if files are in both.
+        """
+
+        results = {}
+        scan_pass = 0
+        _basedir = self.basedir()
+
+        # look in both the inventory base directory and the playbook base directory
+        # unless we do an update for a new playbook base dir
+        if not new_pb_basedir:
+            basedirs = [_basedir, self._playbook_basedir]
+        else:
+            basedirs = [self._playbook_basedir]
+
+        for basedir in basedirs:
+
+            # this can happen from particular API usages, particularly if not run
+            # from /usr/bin/ansible-playbook
+            if basedir is None:
+                continue
+
+            scan_pass = scan_pass + 1
+
+            # it's not an eror if the directory does not exist, keep moving
+            if not os.path.exists(basedir):
+                continue
+
+            # save work of second scan if the directories are the same
+            if _basedir == self._playbook_basedir and scan_pass != 1:
+                continue
+
+            if group and host is None:
+                # load vars in dir/group_vars/name_of_group
+                base_path = os.path.join(basedir, "group_vars/%s" % group.name)
+                results = utils.load_vars(base_path, results, vault_password=self._vault_password)
+
+            elif host and group is None:
+                # same for hostvars in dir/host_vars/name_of_host
+                base_path = os.path.join(basedir, "host_vars/%s" % host.name)
+                results = utils.load_vars(base_path, results, vault_password=self._vault_password)
+
+        # all done, results is a dictionary of variables for this particular host.
+        return results
 

lib/ansible/inventory/vars_plugins/group_vars.py

@@ -1,195 +0,0 @@
-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import stat
-import errno
-
-from ansible import errors
-from ansible import utils
-import ansible.constants as C
-
-def _load_vars(basepath, results, vault_password=None):
-    """
-    Load variables from any potential yaml filename combinations of basepath,
-    returning result.
-    """
-
-    paths_to_check = [ "".join([basepath, ext]) 
-                       for ext in C.YAML_FILENAME_EXTENSIONS ]
-
-    found_paths = []
-
-    for path in paths_to_check:
-        found, results = _load_vars_from_path(path, results, vault_password=vault_password)
-        if found:
-            found_paths.append(path)
-
-
-    # disallow the potentially confusing situation that there are multiple
-    # variable files for the same name. For example if both group_vars/all.yml
-    # and group_vars/all.yaml
-    if len(found_paths) > 1:
-        raise errors.AnsibleError("Multiple variable files found. "
-            "There should only be one. %s" % ( found_paths, ))
-
-    return results
-
-def _load_vars_from_path(path, results, vault_password=None):
-    """
-    Robustly access the file at path and load variables, carefully reporting
-    errors in a friendly/informative way.
-
-    Return the tuple (found, new_results, )
-    """
-
-    try:
-        # in the case of a symbolic link, we want the stat of the link itself,
-        # not its target
-        pathstat = os.lstat(path)
-    except os.error, err:
-        # most common case is that nothing exists at that path.
-        if err.errno == errno.ENOENT:
-            return False, results
-        # otherwise this is a condition we should report to the user
-        raise errors.AnsibleError(
-            "%s is not accessible: %s." 
-            " Please check its permissions." % ( path, err.strerror))
-
-    # symbolic link
-    if stat.S_ISLNK(pathstat.st_mode):
-        try:
-            target = os.path.realpath(path)
-        except os.error, err2:
-            raise errors.AnsibleError("The symbolic link at %s "
-                "is not readable: %s.  Please check its permissions."
-                % (path, err2.strerror, ))
-        # follow symbolic link chains by recursing, so we repeat the same
-        # permissions checks above and provide useful errors.
-        return _load_vars_from_path(target, results)
-
-    # directory
-    if stat.S_ISDIR(pathstat.st_mode):
-
-        # support organizing variables across multiple files in a directory
-        return True, _load_vars_from_folder(path, results, vault_password=vault_password)
-
-    # regular file
-    elif stat.S_ISREG(pathstat.st_mode):
-        data = utils.parse_yaml_from_file(path, vault_password=vault_password)
-        if data and type(data) != dict:
-            raise errors.AnsibleError("%s must be stored as a dictionary/hash" % path)
-        elif data is None:
-            data = {}
-        # combine vars overrides by default but can be configured to do a
-        # hash merge in settings
-        results = utils.combine_vars(results, data)
-        return True, results
-
-    # something else? could be a fifo, socket, device, etc.
-    else:
-        raise errors.AnsibleError("Expected a variable file or directory "
-            "but found a non-file object at path %s" % (path, ))
-
-def _load_vars_from_folder(folder_path, results, vault_password=None):
-    """
-    Load all variables within a folder recursively.
-    """
-
-    # this function and _load_vars_from_path are mutually recursive
-
-    try:
-        names = os.listdir(folder_path)
-    except os.error, err:
-        raise errors.AnsibleError(
-            "This folder cannot be listed: %s: %s." 
-             % ( folder_path, err.strerror))
-        
-    # evaluate files in a stable order rather than whatever order the
-    # filesystem lists them.
-    names.sort() 
-
-    # do not parse hidden files or dirs, e.g. .svn/
-    paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')]
-    for path in paths:
-        _found, results = _load_vars_from_path(path, results, vault_password=vault_password)
-    return results
-
-            
-class VarsModule(object):
-
-    """
-    Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel
-    to the inventory base directory or in the same directory as the playbook.  Variables in the playbook
-    dir will win over the inventory dir if files are in both.
-    """
-
-    def __init__(self, inventory):
-
-        """ constructor """
-
-        self.inventory = inventory
-
-    def run(self, host, vault_password=None):
-
-        """ main body of the plugin, does actual loading """
-
-        inventory = self.inventory
-        basedir = inventory.playbook_basedir()
-        if basedir is not None: 
-            basedir = os.path.abspath(basedir)
-        self.pb_basedir = basedir
-
-        # sort groups by depth so deepest groups can override the less deep ones
-        groupz = sorted(inventory.groups_for_host(host.name), key=lambda g: g.depth)
-        groups = [ g.name for g in groupz ]
-        inventory_basedir = inventory.basedir()
-
-        results = {}
-        scan_pass = 0
-
-        # look in both the inventory base directory and the playbook base directory
-        for basedir in [ inventory_basedir, self.pb_basedir ]:
-
-
-            # this can happen from particular API usages, particularly if not run
-            # from /usr/bin/ansible-playbook
-            if basedir is None:
-                continue
-
-            scan_pass = scan_pass + 1
-
-            # it's not an eror if the directory does not exist, keep moving
-            if not os.path.exists(basedir):
-                continue
-
-            # save work of second scan if the directories are the same
-            if inventory_basedir == self.pb_basedir and scan_pass != 1:
-                continue
-
-            # load vars in dir/group_vars/name_of_group
-            for group in groups:
-                base_path = os.path.join(basedir, "group_vars/%s" % group)
-                results = _load_vars(base_path, results, vault_password=vault_password)
-
-            # same for hostvars in dir/host_vars/name_of_host
-            base_path = os.path.join(basedir, "host_vars/%s" % host.name)
-            results = _load_vars(base_path, results, vault_password=vault_password)
-
-        # all done, results is a dictionary of variables for this particular host.
-        return results
-

lib/ansible/inventory/vars_plugins/noop.py

@@ -0,0 +1,48 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2014, Serge van Ginderachter <serge@vanginderachter.be>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+class VarsModule(object):
+
+    """
+    Loads variables for groups and/or hosts
+    """
+
+    def __init__(self, inventory):
+
+        """ constructor """
+
+        self.inventory = inventory
+        self.inventory_basedir = inventory.basedir()
+
+
+    def run(self, host, vault_password=None):
+        """ For backwards compatibility, when only vars per host were retrieved
+            This method should return both host specific vars as well as vars
+            calculated from groups it is a member of """
+        return {}
+
+
+    def get_host_vars(self, host, vault_password=None):
+        """ Get host specific variables. """
+        return {}
+
+
+    def get_group_vars(self, group, vault_password=None):
+        """ Get group specific variables. """
+        return {}
+

lib/ansible/playbook/__init__.py

@@ -164,6 +164,10 @@ class PlayBook(object):
 
         self.basedir     = os.path.dirname(playbook) or '.'
         utils.plugins.push_basedir(self.basedir)
+
+        # let inventory know the playbook basedir so it can load more vars
+        self.inventory.set_playbook_basedir(self.basedir)
+
         vars = extra_vars.copy()
         vars['playbook_dir'] = self.basedir
         if self.inventory.basedir() is not None:

lib/ansible/utils/__init__.py

@@ -1,4 +1,4 @@
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
 #
 # This file is part of Ansible
 #
@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
+import errno
 import sys
 import re
 import os
@@ -620,18 +621,19 @@ def merge_hash(a, b):
     ''' recursively merges hash b into a
     keys from b take precedence over keys from a '''
 
-    result = copy.deepcopy(a)
+    result = {}
 
-    # next, iterate over b keys and values
+    for dicts in a, b:
-    for k, v in b.iteritems():
+        # next, iterate over b keys and values
-        # if there's already such key in a
+        for k, v in dicts.iteritems():
-        # and that key contains dict
+            # if there's already such key in a
-        if k in result and isinstance(result[k], dict):
+            # and that key contains dict
-            # merge those dicts recursively
+            if k in result and isinstance(result[k], dict):
-            result[k] = merge_hash(a[k], v)
+                # merge those dicts recursively
-        else:
+                result[k] = merge_hash(a[k], v)
-            # otherwise, just copy a value from b to a
+            else:
-            result[k] = v
+                # otherwise, just copy a value from b to a
+                result[k] = v
 
     return result
 
@@ -1208,5 +1210,112 @@ def before_comment(msg):
     msg = msg.replace("**NOT_A_COMMENT**","#")
     return msg
 
+def load_vars(basepath, results, vault_password=None):
+    """
+    Load variables from any potential yaml filename combinations of basepath,
+    returning result.
+    """
+
+    paths_to_check = [ "".join([basepath, ext])
+                       for ext in C.YAML_FILENAME_EXTENSIONS ]
+
+    found_paths = []
+
+    for path in paths_to_check:
+        found, results = _load_vars_from_path(path, results, vault_password=vault_password)
+        if found:
+            found_paths.append(path)
 
 
+    # disallow the potentially confusing situation that there are multiple
+    # variable files for the same name. For example if both group_vars/all.yml
+    # and group_vars/all.yaml
+    if len(found_paths) > 1:
+        raise errors.AnsibleError("Multiple variable files found. "
+            "There should only be one. %s" % ( found_paths, ))
+
+    return results
+
+## load variables from yaml files/dirs
+#  e.g. host/group_vars
+#
+def _load_vars_from_path(path, results, vault_password=None):
+    """
+    Robustly access the file at path and load variables, carefully reporting
+    errors in a friendly/informative way.
+
+    Return the tuple (found, new_results, )
+    """
+
+    try:
+        # in the case of a symbolic link, we want the stat of the link itself,
+        # not its target
+        pathstat = os.lstat(path)
+    except os.error, err:
+        # most common case is that nothing exists at that path.
+        if err.errno == errno.ENOENT:
+            return False, results
+        # otherwise this is a condition we should report to the user
+        raise errors.AnsibleError(
+            "%s is not accessible: %s."
+            " Please check its permissions." % ( path, err.strerror))
+
+    # symbolic link
+    if stat.S_ISLNK(pathstat.st_mode):
+        try:
+            target = os.path.realpath(path)
+        except os.error, err2:
+            raise errors.AnsibleError("The symbolic link at %s "
+                "is not readable: %s.  Please check its permissions."
+                % (path, err2.strerror, ))
+        # follow symbolic link chains by recursing, so we repeat the same
+        # permissions checks above and provide useful errors.
+        return _load_vars_from_path(target, results)
+
+    # directory
+    if stat.S_ISDIR(pathstat.st_mode):
+
+        # support organizing variables across multiple files in a directory
+        return True, _load_vars_from_folder(path, results, vault_password=vault_password)
+
+    # regular file
+    elif stat.S_ISREG(pathstat.st_mode):
+        data = parse_yaml_from_file(path, vault_password=vault_password)
+        if type(data) != dict:
+            raise errors.AnsibleError(
+                "%s must be stored as a dictionary/hash" % path)
+
+        # combine vars overrides by default but can be configured to do a
+        # hash merge in settings
+        results = combine_vars(results, data)
+        return True, results
+
+    # something else? could be a fifo, socket, device, etc.
+    else:
+        raise errors.AnsibleError("Expected a variable file or directory "
+            "but found a non-file object at path %s" % (path, ))
+
+def _load_vars_from_folder(folder_path, results, vault_password=None):
+    """
+    Load all variables within a folder recursively.
+    """
+
+    # this function and _load_vars_from_path are mutually recursive
+
+    try:
+        names = os.listdir(folder_path)
+    except os.error, err:
+        raise errors.AnsibleError(
+            "This folder cannot be listed: %s: %s."
+             % ( folder_path, err.strerror))
+
+    # evaluate files in a stable order rather than whatever order the
+    # filesystem lists them.
+    names.sort()
+
+    # do not parse hidden files or dirs, e.g. .svn/
+    paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')]
+    for path in paths:
+        _found, results = _load_vars_from_path(path, results, vault_password=vault_password)
+    return results
+