From cc948f339c1befc95820b40ac8f5fb574a3cae2e Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Mon, 17 Sep 2012 14:02:30 +0200 Subject: [PATCH 1/2] Allow including files through variables $FILE{file} will be replaced with the contents of "file" $PIPE{cat file} will be replaced with the output of "cat file" --- lib/ansible/playbook/__init__.py | 2 +- lib/ansible/playbook/play.py | 16 +++---- lib/ansible/playbook/task.py | 4 +- lib/ansible/runner/__init__.py | 10 ++-- lib/ansible/runner/action_plugins/copy.py | 4 +- lib/ansible/runner/action_plugins/fetch.py | 4 +- lib/ansible/runner/action_plugins/template.py | 4 +- lib/ansible/utils.py | 46 +++++++++++++++++-- test/TestUtils.py | 18 +++++++- 9 files changed, 79 insertions(+), 29 deletions(-) diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py index 53d7200110..29bd2743a0 100644 --- a/lib/ansible/playbook/__init__.py +++ b/lib/ansible/playbook/__init__.py @@ -267,7 +267,7 @@ class PlayBook(object): for host, results in results.get('contacted',{}).iteritems(): if results.get('changed', False): for handler_name in task.notify: - self._flag_handler(play.handlers(), utils.template(handler_name, task.module_vars), host) + self._flag_handler(play.handlers(), utils.template(play.basedir, handler_name, task.module_vars), host) # ***************************************************** diff --git a/lib/ansible/playbook/play.py b/lib/ansible/playbook/play.py index 1b507df0ec..e2b57064e7 100644 --- a/lib/ansible/playbook/play.py +++ b/lib/ansible/playbook/play.py @@ -57,7 +57,7 @@ class Play(object): raise errors.AnsibleError('hosts declaration is required') elif isinstance(hosts, list): hosts = ';'.join(hosts) - hosts = utils.template(hosts, playbook.extra_vars) + hosts = utils.template(basedir, hosts, playbook.extra_vars) self._ds = ds self.playbook = playbook self.basedir = basedir @@ -69,7 +69,7 @@ class Play(object): self.vars = self._get_vars() self._tasks = ds.get('tasks', []) self._handlers = ds.get('handlers', []) - self.remote_user = utils.template(ds.get('user', self.playbook.remote_user), playbook.extra_vars) + self.remote_user = utils.template(basedir, ds.get('user', self.playbook.remote_user), playbook.extra_vars) self.remote_port = ds.get('port', self.playbook.remote_port) self.sudo = ds.get('sudo', self.playbook.sudo) self.sudo_user = ds.get('sudo_user', self.playbook.sudo_user) @@ -106,8 +106,8 @@ class Play(object): tokens = shlex.split(x['include']) for t in tokens[1:]: (k,v) = t.split("=", 1) - task_vars[k] = utils.template(v, task_vars) - include_file = utils.template(tokens[0], task_vars) + task_vars[k] = utils.template(self.basedir, v, task_vars) + include_file = utils.template(self.basedir, tokens[0], task_vars) data = utils.parse_yaml_from_file(utils.path_dwim(self.basedir, include_file)) elif type(x) == dict: data = [x] @@ -261,10 +261,10 @@ class Play(object): found = False sequence = [] for real_filename in filename: - filename2 = utils.template(real_filename, self.vars) + filename2 = utils.template(self.basedir, real_filename, self.vars) filename3 = filename2 if host is not None: - filename3 = utils.template(filename2, self.playbook.SETUP_CACHE[host]) + filename3 = utils.template(self.basedir, filename2, self.playbook.SETUP_CACHE[host]) filename4 = utils.path_dwim(self.basedir, filename3) sequence.append(filename4) if os.path.exists(filename4): @@ -294,10 +294,10 @@ class Play(object): else: # just one filename supplied, load it! - filename2 = utils.template(filename, self.vars) + filename2 = utils.template(self.basedir, filename, self.vars) filename3 = filename2 if host is not None: - filename3 = utils.template(filename2, self.playbook.SETUP_CACHE[host]) + filename3 = utils.template(self.basedir, filename2, self.playbook.SETUP_CACHE[host]) filename4 = utils.path_dwim(self.basedir, filename3) if self._has_vars_in(filename4): return diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index e5b660f2c9..69f62f4796 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -103,8 +103,8 @@ class Task(object): # allow the user to list comma delimited tags import_tags = import_tags.split(",") - self.name = utils.template(self.name, self.module_vars) - self.action = utils.template(self.action, self.module_vars) + self.name = utils.template(None, self.name, self.module_vars) + self.action = utils.template(None, self.action, self.module_vars) # handle mutually incompatible options if self.with_items is not None and self.first_available_file is not None: diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index 8f983b87aa..266d80cbe1 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -200,7 +200,7 @@ class Runner(object): cmd = "" if not is_new_style: - args = utils.template(args, inject) + args = utils.template(self.basedir, args, inject) argsfile = self._transfer_str(conn, tmp, 'arguments', args) if async_jid is None: cmd = "%s %s" % (remote_module_path, argsfile) @@ -327,13 +327,13 @@ class Runner(object): for (k,v) in self.module_args.iteritems(): new_args = new_args + "%s='%s' " % (k,v) self.module_args = new_args - self.module_args = utils.template(self.module_args, inject) + self.module_args = utils.template(self.basedir, self.module_args, inject) def _check_conditional(conditional): def is_set(var): return not var.startswith("$") return eval(conditional) - conditional = utils.template(self.conditional, inject) + conditional = utils.template(self.basedir, self.conditional, inject) if not _check_conditional(conditional): result = utils.jsonify(dict(skipped=True)) self.callbacks.on_skipped(host, inject.get('item',None)) @@ -352,7 +352,7 @@ class Runner(object): result = dict(failed=True, msg="FAILED: %s" % str(e)) return ReturnData(host=host, comm_ok=False, result=result) - module_name = utils.template(self.module_name, inject) + module_name = utils.template(self.basedir, self.module_name, inject) tmp = '' if self.module_name != 'raw': @@ -499,7 +499,7 @@ class Runner(object): if module_common.REPLACER in module_data: is_new_style=True module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON) - encoded_args = "\"\"\"%s\"\"\"" % utils.template(self.module_args, inject).replace("\"","\\\"") + encoded_args = "\"\"\"%s\"\"\"" % utils.template(self.basedir, self.module_args, inject).replace("\"","\\\"") module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args) # use the correct python interpreter for the host diff --git a/lib/ansible/runner/action_plugins/copy.py b/lib/ansible/runner/action_plugins/copy.py index 2747ad959b..812a2484c8 100644 --- a/lib/ansible/runner/action_plugins/copy.py +++ b/lib/ansible/runner/action_plugins/copy.py @@ -48,7 +48,7 @@ class ActionModule(object): if 'first_available_file' in inject: found = False for fn in inject.get('first_available_file'): - fn = utils.template(fn, inject) + fn = utils.template(self.runner.basedir, fn, inject) if os.path.exists(fn): source = fn found = True @@ -57,7 +57,7 @@ class ActionModule(object): results=dict(failed=True, msg="could not find src in first_available_file list") return ReturnData(conn=conn, results=results) - source = utils.template(source, inject) + source = utils.template(self.runner.basedir, source, inject) source = utils.path_dwim(self.runner.basedir, source) local_md5 = utils.md5(source) diff --git a/lib/ansible/runner/action_plugins/fetch.py b/lib/ansible/runner/action_plugins/fetch.py index a487094bbc..b0c8161451 100644 --- a/lib/ansible/runner/action_plugins/fetch.py +++ b/lib/ansible/runner/action_plugins/fetch.py @@ -44,9 +44,9 @@ class ActionModule(object): return ReturnData(conn=conn, result=results) # apply templating to source argument - source = utils.template(source, inject) + source = utils.template(self.runner.basedir, source, inject) # apply templating to dest argument - dest = utils.template(dest, inject) + dest = utils.template(self.runner.basedir, dest, inject) # files are saved in dest dir, with a subdir for each host, then the filename dest = "%s/%s/%s" % (utils.path_dwim(self.runner.basedir, dest), conn.host, source) diff --git a/lib/ansible/runner/action_plugins/template.py b/lib/ansible/runner/action_plugins/template.py index 61f2da1471..91a50bdede 100644 --- a/lib/ansible/runner/action_plugins/template.py +++ b/lib/ansible/runner/action_plugins/template.py @@ -51,7 +51,7 @@ class ActionModule(object): if 'first_available_file' in inject: found = False for fn in self.runner.module_vars.get('first_available_file'): - fn = utils.template(fn, inject) + fn = utils.template(self.runner.basedir, fn, inject) if os.path.exists(fn): source = fn found = True @@ -60,7 +60,7 @@ class ActionModule(object): result = dict(failed=True, msg="could not find src in first_available_file list") return ReturnData(conn=conn, comm_ok=False, result=result) - source = utils.template(source, inject) + source = utils.template(self.runner.basedir, source, inject) # template the source data locally & transfer try: diff --git a/lib/ansible/utils.py b/lib/ansible/utils.py index 107d7061f9..3ebf5b3846 100644 --- a/lib/ansible/utils.py +++ b/lib/ansible/utils.py @@ -31,6 +31,7 @@ import time import StringIO import imp import glob +import subprocess VERBOSITY=0 @@ -182,7 +183,7 @@ def _varLookup(name, vars): _KEYCRE = re.compile(r"\$(?P\{){0,1}((?(complex)[\w\.\[\]]+|\w+))(?(complex)\})") def varLookup(varname, vars): - ''' helper function used by varReplace ''' + ''' helper function used by with_items ''' m = _KEYCRE.search(varname) if not m: @@ -206,10 +207,9 @@ def varReplace(raw, vars): # Determine replacement value (if unknown variable then preserve # original) - varname = m.group(2) try: - replacement = unicode(_varLookup(varname, vars)) + replacement = unicode(_varLookup(m.group(2), vars)) except VarNotFoundException: replacement = m.group() @@ -220,7 +220,42 @@ def varReplace(raw, vars): return ''.join(done) -def template(text, vars): +_FILEPIPECRE = re.compile(r"\$(?PFILE|PIPE)\{([^\}]+)\}") +def varReplaceFilesAndPipes(basedir, raw): + done = [] # Completed chunks to return + + while raw: + m = _FILEPIPECRE.search(raw) + if not m: + done.append(raw) + break + + # Determine replacement value (if unknown variable then preserve + # original) + + if m.group(1) == "FILE": + try: + f = open(path_dwim(basedir, m.group(2)), "r") + except IOError: + raise VarNotFoundException() + replacement = f.read() + f.close() + elif m.group(1) == "PIPE": + p = subprocess.Popen(m.group(2), shell=True, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode != 0: + raise VarNotFoundException() + replacement = stdout + + start, end = m.span() + done.append(raw[:start]) # Keep stuff leading up to token + done.append(replacement) # Append replacement value + raw = raw[end:] # Continue with remainder of string + + return ''.join(done) + + +def template(basedir, text, vars): ''' run a text buffer through the templating engine until it no longer changes ''' prev_text = '' @@ -235,6 +270,7 @@ def template(text, vars): raise errors.AnsibleError("template recursion depth exceeded") prev_text = text text = varReplace(unicode(text), vars) + text = varReplaceFilesAndPipes(basedir, text) return text def template_from_file(basedir, path, vars): @@ -251,7 +287,7 @@ def template_from_file(basedir, path, vars): res = t.render(vars) if data.endswith('\n') and not res.endswith('\n'): res = res + '\n' - return template(res, vars) + return template(basedir, res, vars) def parse_yaml(data): ''' convert a yaml string to a data structure ''' diff --git a/test/TestUtils.py b/test/TestUtils.py index 00eee68fcf..1f5c9b6e74 100644 --- a/test/TestUtils.py +++ b/test/TestUtils.py @@ -16,7 +16,7 @@ class TestUtils(unittest.TestCase): } } - res = ansible.utils._varLookup('data.who', vars) + res = ansible.utils.varLookup('${data.who}', vars) assert sorted(res) == sorted(vars['data']['who']) @@ -209,10 +209,24 @@ class TestUtils(unittest.TestCase): 'person': 'one', } - res = ansible.utils.template(template, vars) + res = ansible.utils.template(None, template, vars) assert res == u'hello oh great one' + def test_varReplace_include(self): + template = 'hello $FILE{world}' + + res = ansible.utils.template("test", template, {}) + + assert res == u'hello world\n' + + def test_varReplace_include_script(self): + template = 'hello $PIPE{echo world}' + + res = ansible.utils.template("test", template, {}) + + assert res == u'hello world\n' + ##################################### ### Template function tests From 4ded8cb3b348a6f2c944f97fdff065f21e94a4e5 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Tue, 18 Sep 2012 21:34:55 -0400 Subject: [PATCH 2/2] Change $FILE{} and $PIPE{} to $FILE() and $PIPE() --- lib/ansible/utils.py | 2 +- test/TestUtils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ansible/utils.py b/lib/ansible/utils.py index 3ebf5b3846..6fefd09164 100644 --- a/lib/ansible/utils.py +++ b/lib/ansible/utils.py @@ -220,7 +220,7 @@ def varReplace(raw, vars): return ''.join(done) -_FILEPIPECRE = re.compile(r"\$(?PFILE|PIPE)\{([^\}]+)\}") +_FILEPIPECRE = re.compile(r"\$(?PFILE|PIPE)\(([^\}]+)\)") def varReplaceFilesAndPipes(basedir, raw): done = [] # Completed chunks to return diff --git a/test/TestUtils.py b/test/TestUtils.py index 1f5c9b6e74..432f56f62d 100644 --- a/test/TestUtils.py +++ b/test/TestUtils.py @@ -214,14 +214,14 @@ class TestUtils(unittest.TestCase): assert res == u'hello oh great one' def test_varReplace_include(self): - template = 'hello $FILE{world}' + template = 'hello $FILE(world)' res = ansible.utils.template("test", template, {}) assert res == u'hello world\n' def test_varReplace_include_script(self): - template = 'hello $PIPE{echo world}' + template = 'hello $PIPE(echo world)' res = ansible.utils.template("test", template, {})