From b8d7b5041b5e068a2c38a7e7ecf5b8f3e73cda1a Mon Sep 17 00:00:00 2001 From: Jan-Piet Mens Date: Fri, 20 Jul 2012 12:57:15 +0200 Subject: [PATCH] new module: get_url get module (with new module-magic-code!) Usage: ansible -m get -a "url=http://xxxxxxx dest=fileordirctory" all cleanups as per @mpdehaan's suggestions add daisychain added example playbook (get_url.yml) with URLencode example --- examples/playbooks/get_url.yml | 16 ++ library/get_url | 263 +++++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 examples/playbooks/get_url.yml create mode 100755 library/get_url diff --git a/examples/playbooks/get_url.yml b/examples/playbooks/get_url.yml new file mode 100644 index 0000000000..42e07ddd4e --- /dev/null +++ b/examples/playbooks/get_url.yml @@ -0,0 +1,16 @@ +--- +- hosts: webservers + vars: + - jquery_directory: /var/www/html/javascript + - person: 'Susie%20Smith' + tasks: + - name: Create directory for jQuery + action: file dest=${jquery_directory} state=directory mode=0755 + - name: Grab a bunch of jQuery stuff + action: get_url url=http://code.jquery.com/$item dest=${jquery_directory} mode=0444 + with_items: + - jquery.min.js + - mobile/latest/jquery.mobile.min.js + - ui/jquery-ui-git.css + - name: Pass urlencoded name to CGI + action: get_url url=http://example.com/name.cgi?name='${person}' dest=/tmp/test diff --git a/library/get_url b/library/get_url new file mode 100755 index 0000000000..705dd451eb --- /dev/null +++ b/library/get_url @@ -0,0 +1,263 @@ +#!/usr/bin/python + +# (c) 2012, Jan-Piet Mens +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# +# Synopsis: +# ansible -m get_url -a "url=http://some.place/some.file dest=/tmp/file" +# +# Arguments: +# url= (mandatory, no default) +# dest= (mandatory, no default) +# if dest= is a file, url is copied to that file +# if dest= is a directory, determine name from url= and store it in dest/ +# mode, owner, group, ... from the "file" module are also supported +# +# Playbook: +# The dest= feature lets you do this in a Playbook: +# +# - name: Grab a bunch of jQuery stuff +# action: get_url url=http://code.jquery.com/$item dest=${jquery_directory} mode=0444 +# with_items: +# - jquery.min.js +# - mobile/latest/jquery.mobile.min.js +# - ui/jquery-ui-git.css +# +# TODO: +# timeout= +# Support gzip compression? +# http://www.diveintopython.net/http_web_services/gzip_compression.html + +import sys +import os +import shlex +import shutil +import syslog +import datetime +import tempfile + +try: + from hashlib import md5 as _md5 +except ImportError: + from md5 import md5 as _md5 + +HAS_URLLIB2=True +try: + import urllib2 +except ImportError: + HAS_URLLIB2=False +HAS_URLPARSE=True +try: + import urlparse + import socket +except ImportError: + HAS_URLPARSE=False + +# ============================================================== +# support + +def md5(filename): + ''' Return MD5 hex digest of local file, or None if file is not present. ''' + if not os.path.exists(filename): + return None + digest = _md5() + blocksize = 64 * 1024 + infile = open(filename, 'rb') + block = infile.read(blocksize) + while block: + digest.update(block) + block = infile.read(blocksize) + infile.close() + return digest.hexdigest() + +# ============================================================== +# url handling + +def url_filename(url): + return os.path.basename(urlparse.urlsplit(url)[2]) + +def url_do_get(url, dest): + """Get url and return request and info + Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp + """ + USERAGENT = 'ansible-httpget' + info = {} + info['url'] = url + r = None + + if dest: + if os.path.isdir(dest): + destpath = "%s/%s" % (dest, url_filename(url)) + else: + destpath = dest + else: + destpath = url_filename(url) + + info['destpath'] = destpath + + request = urllib2.Request(url) + request.add_header('User-agent', USERAGENT) + + if os.path.exists(destpath): + t = datetime.datetime.utcfromtimestamp(os.path.getmtime(destpath)) + tstamp = t.strftime('%a, %d %b %Y %H:%M:%S +0000') + request.add_header('If-Modified-Since', tstamp) + + try: + r = urllib2.urlopen(request) + + dinfo = dict(r.info()) + for x in dinfo: + info[x] = dinfo[x] + + info['msg'] = "OK %s octets" % r.headers.get('Content-Length', 'unknown') + info['status'] = 200 + except urllib2.HTTPError as e: + # Must not fail_json() here so caller can handle HTTP 304 unmodified + info['msg'] = "%s" % e + info['status'] = e.code + return r, info + except urllib2.URLError as e: + if 'code' in e: + co = e.code + else: + co = -1 + resp = "%s" % e + module.fail_json(msg="Request failed", status_code=co, response=resp) + + return r, info + +def url_get(url, dest): + """Get url and store at dest. If dest is a directory, determine filename + from url, otherwise dest is a file + Return info about the request. + """ + + req, info = url_do_get(url, dest) + + # TODO: should really handle 304, but how? src file could exist (and be + # newer) but be empty ... + + if info['status'] == 304: + module.exit_json(url=url, dest=info.get('destpath', dest), changed=False, msg=info.get('msg', '')) + + # We have the data. Create a temporary file and copy content into that + # to do the MD5-thing + + if info['status'] == 200: + destpath = info['destpath'] + + fd, tempname = tempfile.mkstemp() + f = os.fdopen(fd, 'wb') + try: + shutil.copyfileobj(req, f) + except Exception, err: + os.remove(tempname) + module.fail_json(msg="failed to create temporary content file: %s" % str(err)) + f.close() + req.close() + + return tempname, info + else: + module.fail_json(msg="Request failed", status_code=info['status'], response=info['msg'], url=url) + +# ============================================================== +# main + +def main(): + global module + module = AnsibleModule( + argument_spec = dict( + url = dict(required=True), + dest = dict(required=True), + ) + ) + + url = module.params.get('url', '') + dest = module.params.get('dest', '') + + if url == "": + module.fail_json(msg="url= URL missing") + if dest == "": + module.fail_json(msg="dest= missing") + + dest = os.path.expanduser(dest) + + if not HAS_URLLIB2: + module.fail_json(msg="urllib2 is not installed") + if not HAS_URLPARSE: + module.fail_json(msg="urlparse is not installed") + + + # Here we go... if this succeeds, tmpsrc is the name of a temporary file + # containing slurped content. If it fails, we've already raised an error + # to Ansible + + tmpsrc, info = url_get(url, dest) + + md5sum_src = None + + dest = info.get('destpath', None) + + # raise an error if there is no tmpsrc file + if not os.path.exists(tmpsrc): + os.remove(tmpsrc) + module.fail_json(msg="Request failed", status_code=info['status'], response=info['msg']) + if not os.access(tmpsrc, os.R_OK): + os.remove(tmpsrc) + module.fail_json( msg="Source %s not readable" % (tmpsrc)) + md5sum_src = md5(tmpsrc) + + + md5sum_dest = None + # check if there is no dest file + if os.path.exists(dest): + # raise an error if copy has no permission on dest + if not os.access(dest, os.W_OK): + os.remove(tmpsrc) + module.fail_json( msg="Destination %s not writable" % (dest)) + if not os.access(dest, os.R_OK): + os.remove(tmpsrc) + module.fail_json( msg="Destination %s not readable" % (dest)) + md5sum_dest = md5(dest) + else: + if not os.access(os.path.dirname(dest), os.W_OK): + os.remove(tmpsrc) + module.fail_json( msg="Destination %s not writable" % (os.path.dirname(dest))) + + if md5sum_src != md5sum_dest: + # was os.system("cp %s %s" % (src, dest)) + try: + shutil.copyfile(tmpsrc, dest) + except Exception, err: + os.remove(tmpsrc) + module.fail_json(msg="failed to copy %s to %s: %s" % (tmpsrc, dest, str(err))) + changed = True + else: + changed = False + + # Mission complete + + os.remove(tmpsrc) + module.exit_json(url=url, dest=dest, src=tmpsrc, + md5sum=md5sum_src, changed=changed, msg=info.get('msg', ''), + daisychain="file") + +# this is magic, see lib/ansible/module_common.py +#<> + +main()