From 0b891fc8fb489b2dcee0f09b41afa52c8f8aeb81 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sun, 22 Jul 2012 11:08:16 -0400 Subject: [PATCH] Tweaking daisychain internals to allow get_url to modify the path destination when downloading to a directory. Minor module refactoring. --- examples/playbooks/get_url.yml | 4 +- lib/ansible/module_common.py | 9 +- lib/ansible/runner/__init__.py | 10 ++ library/get_url | 213 +++++++++++---------------------- 4 files changed, 90 insertions(+), 146 deletions(-) diff --git a/examples/playbooks/get_url.yml b/examples/playbooks/get_url.yml index 42e07ddd4e..4800402095 100644 --- a/examples/playbooks/get_url.yml +++ b/examples/playbooks/get_url.yml @@ -12,5 +12,5 @@ - jquery.min.js - mobile/latest/jquery.mobile.min.js - ui/jquery-ui-git.css - - name: Pass urlencoded name to CGI - action: get_url url=http://example.com/name.cgi?name='${person}' dest=/tmp/test + #- name: Pass urlencoded name to CGI + # action: get_url url=http://example.com/name.cgi?name='${person}' dest=/tmp/test diff --git a/lib/ansible/module_common.py b/lib/ansible/module_common.py index 200790f037..2f6452aa12 100644 --- a/lib/ansible/module_common.py +++ b/lib/ansible/module_common.py @@ -128,22 +128,27 @@ class AnsibleModule(object): log_args = re.sub(r'password=.+ (.*)', r"password=NOT_LOGGING_PASSWORD \1", self.args) syslog.syslog(syslog.LOG_NOTICE, 'Invoked with %s' % log_args) + def jsonify(self, data): + return json.dumps(data) + def exit_json(self, **kwargs): ''' return from the module, without error ''' - print json.dumps(kwargs) + print self.jsonify(kwargs) sys.exit(0) def fail_json(self, **kwargs): ''' return from the module, with an error message ''' assert 'msg' in kwargs, "implementation error -- msg to explain the error is required" kwargs['failed'] = True - print json.dumps(kwargs) + print self.jsonify(kwargs) sys.exit(1) def md5(self, filename): ''' Return MD5 hex digest of local file, or None if file is not present. ''' if not os.path.exists(filename): return None + if os.path.isdir(filename): + self.fail_json(msg="attempted to take md5sum of directory: %s" % filename) digest = _md5() blocksize = 64 * 1024 infile = open(filename, 'rb') diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index cd1d0e43d3..30583a8750 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -537,6 +537,14 @@ class Runner(object): group_hosts[g.name] = [ h.name for h in g.hosts ] inject['groups'] = group_hosts + # allow module args to work as a dictionary + # though it is usually a string + new_args = "" + if type(self.module_args) == dict: + for (k,v) in self.module_args.iteritems(): + new_args = new_args + "%s='%s' " % (k,v) + self.module_args = new_args + conditional = utils.template(self.conditional, inject) if not eval(conditional): result = utils.jsonify(dict(skipped=True)) @@ -568,6 +576,8 @@ class Runner(object): if result.is_successful() and 'daisychain' in result.result: chained = True self.module_name = result.result['daisychain'] + if 'daisychain_args' in result.result: + self.module_args = result.result['daisychain_args'] result2 = self._executor_internal_inner(host, inject, port) changed = result.result.get('changed',False) or result2.result.get('changed',False) result.result.update(result2.result) diff --git a/library/get_url b/library/get_url index 705dd451eb..8bf3cbfbdd 100755 --- a/library/get_url +++ b/library/get_url @@ -17,169 +17,118 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . # -# Synopsis: -# ansible -m get_url -a "url=http://some.place/some.file dest=/tmp/file" -# -# Arguments: -# url= (mandatory, no default) -# dest= (mandatory, no default) -# if dest= is a file, url is copied to that file -# if dest= is a directory, determine name from url= and store it in dest/ -# mode, owner, group, ... from the "file" module are also supported -# -# Playbook: -# The dest= feature lets you do this in a Playbook: -# -# - name: Grab a bunch of jQuery stuff -# action: get_url url=http://code.jquery.com/$item dest=${jquery_directory} mode=0444 -# with_items: -# - jquery.min.js -# - mobile/latest/jquery.mobile.min.js -# - ui/jquery-ui-git.css -# -# TODO: -# timeout= -# Support gzip compression? -# http://www.diveintopython.net/http_web_services/gzip_compression.html +# see examples/playbooks/get_url.yml -import sys import os -import shlex import shutil import syslog import datetime import tempfile -try: - from hashlib import md5 as _md5 -except ImportError: - from md5 import md5 as _md5 - HAS_URLLIB2=True try: import urllib2 except ImportError: HAS_URLLIB2=False HAS_URLPARSE=True + try: import urlparse import socket except ImportError: HAS_URLPARSE=False -# ============================================================== -# support - -def md5(filename): - ''' Return MD5 hex digest of local file, or None if file is not present. ''' - if not os.path.exists(filename): - return None - digest = _md5() - blocksize = 64 * 1024 - infile = open(filename, 'rb') - block = infile.read(blocksize) - while block: - digest.update(block) - block = infile.read(blocksize) - infile.close() - return digest.hexdigest() - # ============================================================== # url handling def url_filename(url): - return os.path.basename(urlparse.urlsplit(url)[2]) + fn = os.path.basename(urlparse.urlsplit(url)[2]) + if fn == '': + return 'index.html' + return fn -def url_do_get(url, dest): - """Get url and return request and info - Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp +def url_do_get(module, url, dest): """ + Get url and return request and info + Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp + """ + USERAGENT = 'ansible-httpget' - info = {} - info['url'] = url + info = dict(url=url) r = None + actualdest = None - if dest: - if os.path.isdir(dest): - destpath = "%s/%s" % (dest, url_filename(url)) - else: - destpath = dest + if os.path.isdir(dest): + urlfilename = url_filename(url) + actualdest = "%s/%s" % (dest, url_filename(url)) + module.params['path'] = actualdest else: - destpath = url_filename(url) - - info['destpath'] = destpath + actualdest = dest + info['daisychain_args'] = module.params + info['actualdest'] = actualdest request = urllib2.Request(url) request.add_header('User-agent', USERAGENT) - if os.path.exists(destpath): - t = datetime.datetime.utcfromtimestamp(os.path.getmtime(destpath)) + if os.path.exists(actualdest): + t = datetime.datetime.utcfromtimestamp(os.path.getmtime(actualdest)) tstamp = t.strftime('%a, %d %b %Y %H:%M:%S +0000') request.add_header('If-Modified-Since', tstamp) try: r = urllib2.urlopen(request) - - dinfo = dict(r.info()) - for x in dinfo: - info[x] = dinfo[x] - - info['msg'] = "OK %s octets" % r.headers.get('Content-Length', 'unknown') - info['status'] = 200 + info.update(r.info()) + info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200)) except urllib2.HTTPError as e: # Must not fail_json() here so caller can handle HTTP 304 unmodified - info['msg'] = "%s" % e - info['status'] = e.code + info.update(dict(msg=str(e), status=e.code)) return r, info except urllib2.URLError as e: - if 'code' in e: - co = e.code - else: - co = -1 - resp = "%s" % e - module.fail_json(msg="Request failed", status_code=co, response=resp) + code = getattr(e, 'code', -1) + module.fail_json(msg="Request failed: %s" % str(e), status_code=code) return r, info -def url_get(url, dest): - """Get url and store at dest. If dest is a directory, determine filename - from url, otherwise dest is a file - Return info about the request. +def url_get(module, url, dest): + """ + Download url and store at dest. + If dest is a directory, determine filename from url. + Return (tempfile, info about the request) """ - req, info = url_do_get(url, dest) - - # TODO: should really handle 304, but how? src file could exist (and be - # newer) but be empty ... + req, info = url_do_get(module, url, dest) + # TODO: should really handle 304, but how? src file could exist (and be newer) but empty if info['status'] == 304: - module.exit_json(url=url, dest=info.get('destpath', dest), changed=False, msg=info.get('msg', '')) + module.exit_json(url=url, dest=info.get('actualdest', dest), changed=False, msg=info.get('msg', '')) - # We have the data. Create a temporary file and copy content into that - # to do the MD5-thing - - if info['status'] == 200: - destpath = info['destpath'] - - fd, tempname = tempfile.mkstemp() - f = os.fdopen(fd, 'wb') - try: - shutil.copyfileobj(req, f) - except Exception, err: - os.remove(tempname) - module.fail_json(msg="failed to create temporary content file: %s" % str(err)) - f.close() - req.close() - - return tempname, info - else: + # create a temporary file and copy content to do md5-based replacement + if info['status'] != 200: module.fail_json(msg="Request failed", status_code=info['status'], response=info['msg'], url=url) + actualdest = info['actualdest'] + + fd, tempname = tempfile.mkstemp() + f = os.fdopen(fd, 'wb') + try: + shutil.copyfileobj(req, f) + except Exception, err: + os.remove(tempname) + module.fail_json(msg="failed to create temporary content file: %s" % str(err)) + f.close() + req.close() + return tempname, info # ============================================================== # main def main(): - global module + + # does this really happen on non-ancient python? + if not HAS_URLLIB2: + module.fail_json(msg="urllib2 is not installed") + if not HAS_URLPARSE: + module.fail_json(msg="urlparse is not installed") + module = AnsibleModule( argument_spec = dict( url = dict(required=True), @@ -187,31 +136,14 @@ def main(): ) ) - url = module.params.get('url', '') - dest = module.params.get('dest', '') + url = module.params['url'] + dest = os.path.expanduser(module.params['dest']) - if url == "": - module.fail_json(msg="url= URL missing") - if dest == "": - module.fail_json(msg="dest= missing") - - dest = os.path.expanduser(dest) - - if not HAS_URLLIB2: - module.fail_json(msg="urllib2 is not installed") - if not HAS_URLPARSE: - module.fail_json(msg="urlparse is not installed") - - - # Here we go... if this succeeds, tmpsrc is the name of a temporary file - # containing slurped content. If it fails, we've already raised an error - # to Ansible - - tmpsrc, info = url_get(url, dest) - - md5sum_src = None - - dest = info.get('destpath', None) + # download to tmpsrc + tmpsrc, info = url_get(module, url, dest) + md5sum_src = None + md5sum_dest = None + dest = info['actualdest'] # raise an error if there is no tmpsrc file if not os.path.exists(tmpsrc): @@ -220,10 +152,8 @@ def main(): if not os.access(tmpsrc, os.R_OK): os.remove(tmpsrc) module.fail_json( msg="Source %s not readable" % (tmpsrc)) - md5sum_src = md5(tmpsrc) - - - md5sum_dest = None + md5sum_src = module.md5(tmpsrc) + # check if there is no dest file if os.path.exists(dest): # raise an error if copy has no permission on dest @@ -233,14 +163,13 @@ def main(): if not os.access(dest, os.R_OK): os.remove(tmpsrc) module.fail_json( msg="Destination %s not readable" % (dest)) - md5sum_dest = md5(dest) + md5sum_dest = module.md5(dest) else: if not os.access(os.path.dirname(dest), os.W_OK): os.remove(tmpsrc) module.fail_json( msg="Destination %s not writable" % (os.path.dirname(dest))) if md5sum_src != md5sum_dest: - # was os.system("cp %s %s" % (src, dest)) try: shutil.copyfile(tmpsrc, dest) except Exception, err: @@ -250,12 +179,12 @@ def main(): else: changed = False - # Mission complete - os.remove(tmpsrc) - module.exit_json(url=url, dest=dest, src=tmpsrc, - md5sum=md5sum_src, changed=changed, msg=info.get('msg', ''), - daisychain="file") + + # Mission complete + module.exit_json(url=url, dest=dest, src=tmpsrc, md5sum=md5sum_src, + changed=changed, msg=info.get('msg',''), + daisychain="file", daisychain_args=info.get('daisychain_args','')) # this is magic, see lib/ansible/module_common.py #<>