diff --git a/bin/ansible b/bin/ansible index 1e2540fafb..7e767b2f7d 100755 --- a/bin/ansible +++ b/bin/ansible @@ -136,7 +136,7 @@ class Cli(object): if not options.ask_vault_pass: vault_pass = tmp_vault_pass - inventory_manager = inventory.Inventory(options.inventory) + inventory_manager = inventory.Inventory(options.inventory, vault_password=vault_pass) if options.subset: inventory_manager.subset(options.subset) hosts = inventory_manager.list_hosts(pattern) diff --git a/bin/ansible-playbook b/bin/ansible-playbook index d7c9182e2f..149a9f1c6e 100755 --- a/bin/ansible-playbook +++ b/bin/ansible-playbook @@ -100,11 +100,6 @@ def main(args): if (options.ask_vault_pass and options.vault_password_file): parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive") - inventory = ansible.inventory.Inventory(options.inventory) - inventory.subset(options.subset) - if len(inventory.list_hosts()) == 0: - raise errors.AnsibleError("provided hosts list is empty") - sshpass = None sudopass = None su_pass = None @@ -160,12 +155,14 @@ def main(args): if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook) + inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass) + inventory.subset(options.subset) + if len(inventory.list_hosts()) == 0: + raise errors.AnsibleError("provided hosts list is empty") + # run all playbooks specified on the command line for playbook in args: - # let inventory know which playbooks are using so it can know the basedirs - inventory.set_playbook_basedir(os.path.dirname(playbook)) - stats = callbacks.AggregateStats() playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY) if options.step: diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index 7083cf9349..3657ac8146 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -38,13 +38,14 @@ class Inventory(object): __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', 'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list', - '_pattern_cache', '_vars_plugins', '_playbook_basedir'] + '_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir'] - def __init__(self, host_list=C.DEFAULT_HOST_LIST): + def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None): # the host file file, or script path, or list of hosts # if a list, inventory data will NOT be loaded self.host_list = host_list + self._vault_password=vault_password # caching to avoid repeated calculations, particularly with # external inventory scripts. @@ -55,7 +56,7 @@ class Inventory(object): self._groups_list = {} self._pattern_cache = {} - # to be set by calling set_playbook_basedir by ansible-playbook + # to be set by calling set_playbook_basedir by playbook code self._playbook_basedir = None # the inventory object holds a list of groups @@ -139,6 +140,14 @@ class Inventory(object): self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ] + # get group vars from group_vars/ files and vars plugins + for group in self.groups: + group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password)) + + # get host vars from host_vars/ files and vars plugins + for host in self.get_hosts(): + host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password)) + def _match(self, str, pattern_str): if pattern_str.startswith('~'): @@ -392,19 +401,35 @@ class Inventory(object): return group return None - def get_group_variables(self, groupname): - if groupname not in self._vars_per_group: - self._vars_per_group[groupname] = self._get_group_variables(groupname) + def get_group_variables(self, groupname, update_cached=False, vault_password=None): + if groupname not in self._vars_per_group or update_cached: + self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password) return self._vars_per_group[groupname] - def _get_group_variables(self, groupname): + def _get_group_variables(self, groupname, vault_password=None): + group = self.get_group(groupname) if group is None: raise Exception("group not found: %s" % groupname) - return group.get_variables() - def get_variables(self, hostname, vault_password=None): - if hostname not in self._vars_per_host: + vars = {} + + # plugin.get_group_vars retrieves just vars for specific group + vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')] + for updated in vars_results: + if updated is not None: + vars = utils.combine_vars(vars, updated) + + # get group variables set by Inventory Parsers + vars = utils.combine_vars(vars, group.get_variables()) + + # Read group_vars/ files + vars = utils.combine_vars(vars, self.get_group_vars(group)) + + return vars + + def get_variables(self, hostname, update_cached=False, vault_password=None): + if hostname not in self._vars_per_host or update_cached: self._vars_per_host[hostname] = self._get_variables(hostname, vault_password=vault_password) return self._vars_per_host[hostname] @@ -415,14 +440,31 @@ class Inventory(object): raise errors.AnsibleError("host not found: %s" % hostname) vars = {} - vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins ] + + # plugin.run retrieves all vars (also from groups) for host + vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')] for updated in vars_results: if updated is not None: vars = utils.combine_vars(vars, updated) + # plugin.get_host_vars retrieves just vars for specific host + vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')] + for updated in vars_results: + if updated is not None: + vars = utils.combine_vars(vars, updated) + + # get host variables set by Inventory Parsers vars = utils.combine_vars(vars, host.get_variables()) + + # still need to check InventoryParser per host vars + # which actually means InventoryScript per host, + # which is not performant if self.parser is not None: vars = utils.combine_vars(vars, self.parser.get_host_variables(host)) + + # Read host_vars/ files + vars = utils.combine_vars(vars, self.get_host_vars(host)) + return vars def add_group(self, group): @@ -525,10 +567,73 @@ class Inventory(object): return self._playbook_basedir def set_playbook_basedir(self, dir): - """ - sets the base directory of the playbook so inventory plugins can use it to find - variable files and other things. """ - self._playbook_basedir = dir + sets the base directory of the playbook so inventory can use it as a + basedir for host_ and group_vars, and other things. + """ + # Only update things if dir is a different playbook basedir + if dir != self._playbook_basedir: + self._playbook_basedir = dir + # get group vars from group_vars/ files + for group in self.groups: + group.vars = utils.combine_vars(group.vars, self.get_group_vars(group, new_pb_basedir=True)) + # get host vars from host_vars/ files + for host in self.get_hosts(): + host.vars = utils.combine_vars(host.vars, self.get_host_vars(host, new_pb_basedir=True)) + def get_host_vars(self, host, new_pb_basedir=False): + """ Read host_vars/ files """ + return self._get_hostgroup_vars(host=host, group=None, new_pb_basedir=False) + + def get_group_vars(self, group, new_pb_basedir=False): + """ Read group_vars/ files """ + return self._get_hostgroup_vars(host=None, group=group, new_pb_basedir=False) + + def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False): + """ + Loads variables from group_vars/ and host_vars/ in directories parallel + to the inventory base directory or in the same directory as the playbook. Variables in the playbook + dir will win over the inventory dir if files are in both. + """ + + results = {} + scan_pass = 0 + _basedir = self.basedir() + + # look in both the inventory base directory and the playbook base directory + # unless we do an update for a new playbook base dir + if not new_pb_basedir: + basedirs = [_basedir, self._playbook_basedir] + else: + basedirs = [self._playbook_basedir] + + for basedir in basedirs: + + # this can happen from particular API usages, particularly if not run + # from /usr/bin/ansible-playbook + if basedir is None: + continue + + scan_pass = scan_pass + 1 + + # it's not an eror if the directory does not exist, keep moving + if not os.path.exists(basedir): + continue + + # save work of second scan if the directories are the same + if _basedir == self._playbook_basedir and scan_pass != 1: + continue + + if group and host is None: + # load vars in dir/group_vars/name_of_group + base_path = os.path.join(basedir, "group_vars/%s" % group.name) + results = utils.load_vars(base_path, results, vault_password=self._vault_password) + + elif host and group is None: + # same for hostvars in dir/host_vars/name_of_host + base_path = os.path.join(basedir, "host_vars/%s" % host.name) + results = utils.load_vars(base_path, results, vault_password=self._vault_password) + + # all done, results is a dictionary of variables for this particular host. + return results diff --git a/lib/ansible/inventory/vars_plugins/group_vars.py b/lib/ansible/inventory/vars_plugins/group_vars.py deleted file mode 100644 index 96a24318bd..0000000000 --- a/lib/ansible/inventory/vars_plugins/group_vars.py +++ /dev/null @@ -1,195 +0,0 @@ -# (c) 2012-2014, Michael DeHaan -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -import os -import stat -import errno - -from ansible import errors -from ansible import utils -import ansible.constants as C - -def _load_vars(basepath, results, vault_password=None): - """ - Load variables from any potential yaml filename combinations of basepath, - returning result. - """ - - paths_to_check = [ "".join([basepath, ext]) - for ext in C.YAML_FILENAME_EXTENSIONS ] - - found_paths = [] - - for path in paths_to_check: - found, results = _load_vars_from_path(path, results, vault_password=vault_password) - if found: - found_paths.append(path) - - - # disallow the potentially confusing situation that there are multiple - # variable files for the same name. For example if both group_vars/all.yml - # and group_vars/all.yaml - if len(found_paths) > 1: - raise errors.AnsibleError("Multiple variable files found. " - "There should only be one. %s" % ( found_paths, )) - - return results - -def _load_vars_from_path(path, results, vault_password=None): - """ - Robustly access the file at path and load variables, carefully reporting - errors in a friendly/informative way. - - Return the tuple (found, new_results, ) - """ - - try: - # in the case of a symbolic link, we want the stat of the link itself, - # not its target - pathstat = os.lstat(path) - except os.error, err: - # most common case is that nothing exists at that path. - if err.errno == errno.ENOENT: - return False, results - # otherwise this is a condition we should report to the user - raise errors.AnsibleError( - "%s is not accessible: %s." - " Please check its permissions." % ( path, err.strerror)) - - # symbolic link - if stat.S_ISLNK(pathstat.st_mode): - try: - target = os.path.realpath(path) - except os.error, err2: - raise errors.AnsibleError("The symbolic link at %s " - "is not readable: %s. Please check its permissions." - % (path, err2.strerror, )) - # follow symbolic link chains by recursing, so we repeat the same - # permissions checks above and provide useful errors. - return _load_vars_from_path(target, results) - - # directory - if stat.S_ISDIR(pathstat.st_mode): - - # support organizing variables across multiple files in a directory - return True, _load_vars_from_folder(path, results, vault_password=vault_password) - - # regular file - elif stat.S_ISREG(pathstat.st_mode): - data = utils.parse_yaml_from_file(path, vault_password=vault_password) - if data and type(data) != dict: - raise errors.AnsibleError("%s must be stored as a dictionary/hash" % path) - elif data is None: - data = {} - # combine vars overrides by default but can be configured to do a - # hash merge in settings - results = utils.combine_vars(results, data) - return True, results - - # something else? could be a fifo, socket, device, etc. - else: - raise errors.AnsibleError("Expected a variable file or directory " - "but found a non-file object at path %s" % (path, )) - -def _load_vars_from_folder(folder_path, results, vault_password=None): - """ - Load all variables within a folder recursively. - """ - - # this function and _load_vars_from_path are mutually recursive - - try: - names = os.listdir(folder_path) - except os.error, err: - raise errors.AnsibleError( - "This folder cannot be listed: %s: %s." - % ( folder_path, err.strerror)) - - # evaluate files in a stable order rather than whatever order the - # filesystem lists them. - names.sort() - - # do not parse hidden files or dirs, e.g. .svn/ - paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')] - for path in paths: - _found, results = _load_vars_from_path(path, results, vault_password=vault_password) - return results - - -class VarsModule(object): - - """ - Loads variables from group_vars/ and host_vars/ in directories parallel - to the inventory base directory or in the same directory as the playbook. Variables in the playbook - dir will win over the inventory dir if files are in both. - """ - - def __init__(self, inventory): - - """ constructor """ - - self.inventory = inventory - - def run(self, host, vault_password=None): - - """ main body of the plugin, does actual loading """ - - inventory = self.inventory - basedir = inventory.playbook_basedir() - if basedir is not None: - basedir = os.path.abspath(basedir) - self.pb_basedir = basedir - - # sort groups by depth so deepest groups can override the less deep ones - groupz = sorted(inventory.groups_for_host(host.name), key=lambda g: g.depth) - groups = [ g.name for g in groupz ] - inventory_basedir = inventory.basedir() - - results = {} - scan_pass = 0 - - # look in both the inventory base directory and the playbook base directory - for basedir in [ inventory_basedir, self.pb_basedir ]: - - - # this can happen from particular API usages, particularly if not run - # from /usr/bin/ansible-playbook - if basedir is None: - continue - - scan_pass = scan_pass + 1 - - # it's not an eror if the directory does not exist, keep moving - if not os.path.exists(basedir): - continue - - # save work of second scan if the directories are the same - if inventory_basedir == self.pb_basedir and scan_pass != 1: - continue - - # load vars in dir/group_vars/name_of_group - for group in groups: - base_path = os.path.join(basedir, "group_vars/%s" % group) - results = _load_vars(base_path, results, vault_password=vault_password) - - # same for hostvars in dir/host_vars/name_of_host - base_path = os.path.join(basedir, "host_vars/%s" % host.name) - results = _load_vars(base_path, results, vault_password=vault_password) - - # all done, results is a dictionary of variables for this particular host. - return results - diff --git a/lib/ansible/inventory/vars_plugins/noop.py b/lib/ansible/inventory/vars_plugins/noop.py new file mode 100644 index 0000000000..5d4b4b6658 --- /dev/null +++ b/lib/ansible/inventory/vars_plugins/noop.py @@ -0,0 +1,48 @@ +# (c) 2012-2014, Michael DeHaan +# (c) 2014, Serge van Ginderachter +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +class VarsModule(object): + + """ + Loads variables for groups and/or hosts + """ + + def __init__(self, inventory): + + """ constructor """ + + self.inventory = inventory + self.inventory_basedir = inventory.basedir() + + + def run(self, host, vault_password=None): + """ For backwards compatibility, when only vars per host were retrieved + This method should return both host specific vars as well as vars + calculated from groups it is a member of """ + return {} + + + def get_host_vars(self, host, vault_password=None): + """ Get host specific variables. """ + return {} + + + def get_group_vars(self, group, vault_password=None): + """ Get group specific variables. """ + return {} + diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py index 0232f5b86e..4188d65d54 100644 --- a/lib/ansible/playbook/__init__.py +++ b/lib/ansible/playbook/__init__.py @@ -164,6 +164,10 @@ class PlayBook(object): self.basedir = os.path.dirname(playbook) or '.' utils.plugins.push_basedir(self.basedir) + + # let inventory know the playbook basedir so it can load more vars + self.inventory.set_playbook_basedir(self.basedir) + vars = extra_vars.copy() vars['playbook_dir'] = self.basedir if self.inventory.basedir() is not None: diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index 9d09254e3c..fb391f9efd 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -1,4 +1,4 @@ -# (c) 2012, Michael DeHaan +# (c) 2012-2014, Michael DeHaan # # This file is part of Ansible # @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . +import errno import sys import re import os @@ -620,18 +621,19 @@ def merge_hash(a, b): ''' recursively merges hash b into a keys from b take precedence over keys from a ''' - result = copy.deepcopy(a) + result = {} - # next, iterate over b keys and values - for k, v in b.iteritems(): - # if there's already such key in a - # and that key contains dict - if k in result and isinstance(result[k], dict): - # merge those dicts recursively - result[k] = merge_hash(a[k], v) - else: - # otherwise, just copy a value from b to a - result[k] = v + for dicts in a, b: + # next, iterate over b keys and values + for k, v in dicts.iteritems(): + # if there's already such key in a + # and that key contains dict + if k in result and isinstance(result[k], dict): + # merge those dicts recursively + result[k] = merge_hash(a[k], v) + else: + # otherwise, just copy a value from b to a + result[k] = v return result @@ -1208,5 +1210,112 @@ def before_comment(msg): msg = msg.replace("**NOT_A_COMMENT**","#") return msg +def load_vars(basepath, results, vault_password=None): + """ + Load variables from any potential yaml filename combinations of basepath, + returning result. + """ + + paths_to_check = [ "".join([basepath, ext]) + for ext in C.YAML_FILENAME_EXTENSIONS ] + + found_paths = [] + + for path in paths_to_check: + found, results = _load_vars_from_path(path, results, vault_password=vault_password) + if found: + found_paths.append(path) + # disallow the potentially confusing situation that there are multiple + # variable files for the same name. For example if both group_vars/all.yml + # and group_vars/all.yaml + if len(found_paths) > 1: + raise errors.AnsibleError("Multiple variable files found. " + "There should only be one. %s" % ( found_paths, )) + + return results + +## load variables from yaml files/dirs +# e.g. host/group_vars +# +def _load_vars_from_path(path, results, vault_password=None): + """ + Robustly access the file at path and load variables, carefully reporting + errors in a friendly/informative way. + + Return the tuple (found, new_results, ) + """ + + try: + # in the case of a symbolic link, we want the stat of the link itself, + # not its target + pathstat = os.lstat(path) + except os.error, err: + # most common case is that nothing exists at that path. + if err.errno == errno.ENOENT: + return False, results + # otherwise this is a condition we should report to the user + raise errors.AnsibleError( + "%s is not accessible: %s." + " Please check its permissions." % ( path, err.strerror)) + + # symbolic link + if stat.S_ISLNK(pathstat.st_mode): + try: + target = os.path.realpath(path) + except os.error, err2: + raise errors.AnsibleError("The symbolic link at %s " + "is not readable: %s. Please check its permissions." + % (path, err2.strerror, )) + # follow symbolic link chains by recursing, so we repeat the same + # permissions checks above and provide useful errors. + return _load_vars_from_path(target, results) + + # directory + if stat.S_ISDIR(pathstat.st_mode): + + # support organizing variables across multiple files in a directory + return True, _load_vars_from_folder(path, results, vault_password=vault_password) + + # regular file + elif stat.S_ISREG(pathstat.st_mode): + data = parse_yaml_from_file(path, vault_password=vault_password) + if type(data) != dict: + raise errors.AnsibleError( + "%s must be stored as a dictionary/hash" % path) + + # combine vars overrides by default but can be configured to do a + # hash merge in settings + results = combine_vars(results, data) + return True, results + + # something else? could be a fifo, socket, device, etc. + else: + raise errors.AnsibleError("Expected a variable file or directory " + "but found a non-file object at path %s" % (path, )) + +def _load_vars_from_folder(folder_path, results, vault_password=None): + """ + Load all variables within a folder recursively. + """ + + # this function and _load_vars_from_path are mutually recursive + + try: + names = os.listdir(folder_path) + except os.error, err: + raise errors.AnsibleError( + "This folder cannot be listed: %s: %s." + % ( folder_path, err.strerror)) + + # evaluate files in a stable order rather than whatever order the + # filesystem lists them. + names.sort() + + # do not parse hidden files or dirs, e.g. .svn/ + paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')] + for path in paths: + _found, results = _load_vars_from_path(path, results, vault_password=vault_password) + return results +