From ef8ea1349585aba406395aaa0897afc3f6d110f5 Mon Sep 17 00:00:00 2001 From: Jeremy Katz Date: Thu, 23 Feb 2012 20:51:29 -0500 Subject: [PATCH 01/33] Fall back to standalone simplejson module CentOS5 has python 2.4 which doesn't have a built-in json module --- library/command | 5 ++++- library/ping | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/library/command b/library/command index 53a8d6ffff..c11cfebd25 100755 --- a/library/command +++ b/library/command @@ -1,6 +1,9 @@ #!/usr/bin/python -import json +try: + import json +except ImportError: + import simplejson as json import subprocess import sys import datetime diff --git a/library/ping b/library/ping index fe0b394825..e40be68a2f 100644 --- a/library/ping +++ b/library/ping @@ -1,5 +1,8 @@ #!/usr/bin/python -import json +try: + import json +except ImportError: + import simplejson as json print json.dumps(1) From 2c873a44671f206431c9117225049426769f2cc4 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 21:07:03 -0500 Subject: [PATCH 02/33] Adding setup.py --- setup.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..d5664ee55a --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup(name='ansible', + version='1.0', + description='Minimal SSH command and control', + author='Michael DeHaan', + author_email='michael.dehaan@gmail.com', + url='http://github.com/mpdehaan/ansible/', + license='MIT', + package_dir = { 'ansible' : 'lib/ansible' }, + packages=[ + 'ansible', + ], + data_files=[ + ('/usr/share/ancible', 'library/ping'), + ('/usr/share/ancible', 'library/command'), + ('/usr/share/ancible', 'library/facter'), + ('/usr/share/ancible', 'library/copy'), + ], + scripts=[ + 'bin/ansible', + ] +) + From 3da6370a654b20cd90cd1bf492cd032c57e177b2 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 21:37:39 -0500 Subject: [PATCH 03/33] use defaults better, improve/fix setup.py --- bin/ansible | 12 +++---- lib/ansible/__init__.py | 76 ++++++++++++++++++++++++++++++----------- setup.py | 10 +++--- 3 files changed, 66 insertions(+), 32 deletions(-) diff --git a/bin/ansible b/bin/ansible index d6070c2c84..6425dcd714 100755 --- a/bin/ansible +++ b/bin/ansible @@ -26,8 +26,8 @@ import json import os import ansible -DEFAULT_HOST_LIST = '~/.ansible_hosts' -DEFAULT_MODULE_PATH = '~/ansible' +DEFAULT_HOST_LIST = '/etc/ansible/hosts' +DEFAULT_MODULE_PATH = '/usr/share/ansible' DEFAULT_MODULE_NAME = 'ping' DEFAULT_PATTERN = '*' DEFAULT_FORKS = 3 @@ -54,7 +54,6 @@ class Cli(object): help="hostname pattern", default=DEFAULT_PATTERN) options, args = parser.parse_args() - host_list = self._host_list(options.host_list) # TODO: more shell like splitting on module_args would # be a good idea @@ -63,15 +62,12 @@ class Cli(object): module_name=options.module_name, module_path=options.module_path, module_args=options.module_args.split(' '), - host_list=host_list, + host_list=options.host_list, forks=options.forks, pattern=options.pattern, + verbose=False, ) - def _host_list(self, host_list): - host_list = os.path.expanduser(host_list) - return file(host_list).read().split("\n") - if __name__ == '__main__': result = Cli().runner().run() diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index a056727047..4af0ba405a 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -24,19 +24,18 @@ from multiprocessing import Process, Pipe from itertools import izip import os import json +import traceback # non-core import paramiko -# TODO -- library should have defaults, not just CLI -# update Runner constructor below to use - -DEFAULT_HOST_LIST = '~/.ansible_hosts' -DEFAULT_MODULE_PATH = '~/ansible' +DEFAULT_HOST_LIST = '/etc/ansible/hosts' +DEFAULT_MODULE_PATH = '/usr/share/ansible' DEFAULT_MODULE_NAME = 'ping' DEFAULT_PATTERN = '*' DEFAULT_FORKS = 3 DEFAULT_MODULE_ARGS = '' +DEFAULT_TIMEOUT = 60 class Pooler(object): @@ -59,20 +58,40 @@ class Pooler(object): class Runner(object): - def __init__(self, host_list=[], module_path=None, - module_name=None, module_args=[], - forks=3, timeout=60, pattern='*'): + def __init__(self, + host_list=DEFAULT_HOST_LIST, + module_path=DEFAULT_MODULE_PATH, + module_name=DEFAULT_MODULE_NAME, + module_args=DEFAULT_MODULE_ARGS, + forks=DEFAULT_FORKS, + timeout=DEFAULT_TIMEOUT, + pattern=DEFAULT_PATTERN, + verbose=False): + - self.host_list = host_list + ''' + Constructor. + ''' + + self.host_list = self._parse_hosts(host_list) self.module_path = module_path self.module_name = module_name self.forks = forks self.pattern = pattern self.module_args = module_args self.timeout = timeout - + self.verbose = verbose + + def _parse_hosts(self, host_list): + ''' parse the host inventory file if not sent as an array ''' + if type(host_list) != list: + host_list = os.path.expanduser(host_list) + return file(host_list).read().split("\n") + return host_list + def _matches(self, host_name): + ''' returns if a hostname is matched by the pattern ''' if host_name == '': return False if fnmatch.fnmatch(host_name, self.pattern): @@ -80,6 +99,7 @@ class Runner(object): return False def _connect(self, host): + ''' obtains a paramiko connection to the host ''' ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: @@ -87,20 +107,29 @@ class Runner(object): allow_agent=True, look_for_keys=True) return ssh except: + # TODO -- just convert traceback to string + # and return a seperate hash of failed hosts + if self.verbose: + traceback.print_exc() return None def _executor(self, host): + ''' callback executed in parallel for each host ''' # TODO: try/catch returning none + conn = self._connect(host) if not conn: return [ host, None ] + if self.module_name != "copy": + # transfer a module, set it executable, and run it outpath = self._copy_module(conn) self._exec_command(conn, "chmod +x %s" % outpath) cmd = self._command(outpath) result = self._exec_command(conn, cmd) result = json.loads(result) else: + # SFTP file copy module is not really a module ftp = conn.open_sftp() ftp.put(self.module_args[0], self.module_args[1]) ftp.close() @@ -109,23 +138,32 @@ class Runner(object): return [ host, result ] def _command(self, outpath): + ''' form up a command string ''' cmd = "%s %s" % (outpath, " ".join(self.module_args)) return cmd def _exec_command(self, conn, cmd): + ''' execute a command over SSH ''' stdin, stdout, stderr = conn.exec_command(cmd) results = stdout.read() return results def _copy_module(self, conn): - inpath = os.path.expanduser(os.path.join(self.module_path, self.module_name)) - outpath = os.path.join("/var/spool/", "ansible_%s" % self.module_name) - ftp = conn.open_sftp() - ftp.put(inpath, outpath) - ftp.close() - return outpath + ''' transfer a module over SFTP ''' + in_path = os.path.expanduser( + os.path.join(self.module_path, self.module_name) + ) + out_path = os.path.join( + "/var/spool/", + "ansible_%s" % self.module_name + ) + sftp = conn.open_sftp() + sftp.put(in_path, out_path) + sftp.close() + return out_path def run(self): + ''' xfer & run module on all matched hosts ''' hosts = [ h for h in self.host_list if self._matches(h) ] def executor(x): return self._executor(x) @@ -136,12 +174,10 @@ class Runner(object): if __name__ == '__main__': - - # TODO: if host list is string load from file + # test code... r = Runner( - host_list = [ '127.0.0.1' ], - module_path='~/ansible', + host_list = DEFAULT_HOST_LIST, module_name='ping', module_args='', pattern='*', diff --git a/setup.py b/setup.py index d5664ee55a..25f72ee5fd 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,12 @@ setup(name='ansible', 'ansible', ], data_files=[ - ('/usr/share/ancible', 'library/ping'), - ('/usr/share/ancible', 'library/command'), - ('/usr/share/ancible', 'library/facter'), - ('/usr/share/ancible', 'library/copy'), + ('/usr/share/ansible', [ + 'library/ping', + 'library/command', + 'library/facter', + 'library/copy', + ]) ], scripts=[ 'bin/ansible', From 03647d64e9c8dd085f4cd9b67eb197cb72b2e00c Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 21:47:31 -0500 Subject: [PATCH 04/33] Update docs, added TODO.md --- README.md | 78 ++++++++++++++++++++++++++++--------------------------- TODO.md | 15 +++++++++++ 2 files changed, 55 insertions(+), 38 deletions(-) create mode 100644 TODO.md diff --git a/README.md b/README.md index 1ef14a749e..71fc7f5015 100644 --- a/README.md +++ b/README.md @@ -3,24 +3,33 @@ Ansible Ansible is a extra-simple Python API for doing 'remote things' over SSH. -While [Func](http://fedorahosted.org/func), which I co-wrote, aspired to avoid using SSH and have it's own daemon infrastructure, Ansible aspires to be quite different and more minimal, but still able to grow more modularly over time. This is based on talking to a lot of users of various tools and wishing to eliminate problems with connectivity and long running daemons, or not picking tool X because they preferred to code in Y. +While [Func](http://fedorahosted.org/func), which I co-wrote, +aspired to avoid using SSH and have it's own daemon infrastructure, +Ansible aspires to be quite different and more minimal, but still able +to grow more modularly over time. This is based on talking to a lot of +users of various tools and wishing to eliminate problems with connectivity +and long running daemons, or not picking tool X because they preferred to +code in Y. -Why use Ansible versus something else? (Fabric, Capistrano, mCollective, Func, SaltStack, etc?) It will have far less code, it will be more correct, and it will be the easiest thing to hack on and use you'll ever see -- regardless of your favorite language of choice. Want to only code plugins in bash or clojure? Ansible doesn't care. The docs will fit on one page and the source will be blindingly obvious. +Why use Ansible versus something else? (Fabric, Capistrano, mCollective, +Func, SaltStack, etc?) It will have far less code, it will be more correct, +and it will be the easiest thing to hack on and use you'll ever see -- +regardless of your favorite language of choice. Want to only code plugins +in bash or clojure? Ansible doesn't care. The docs will fit on one page +and the source will be blindingly obvious. -Principles -========== +Design Principles +================= * Dead simple setup * Super fast & parallel by default * No server or client daemons, uses existing SSHd * No additional software required on client boxes - * Everything is self updating on the clients. "Modules" are remotely transferred to target boxes and exec'd, and do not stay active or consume resources. - * Only SSH keys are allowed for authentication - * usage of ssh-agent is more or less required (no passwords) - * plugins can be written in ANY language - * as with Func, API usage is an equal citizen to CLI usage - * use Python's multiprocessing capabilities to emulate Func's forkbomb logic - * all file paths can be specified as command line options easily allowing non-root usage + * Everything is self updating on the clients + * Encourages use of ssh-agent + * Plugins can be written in ANY language + * API usage is an equal citizen to CLI usage + * Can be controlled/installed/used as non-root Requirements ============ @@ -33,11 +42,11 @@ For the server the tool is running from, *only*: Inventory file ============== -The inventory file is a required list of hostnames that can be potentially managed by -ansible. Eventually this file may be editable via the CLI, but for now, is -edited with your favorite text editor. +The inventory file is a required list of hostnames that can be +potentially managed by ansible. Eventually this file may be editable +via the CLI, but for now, is edited with your favorite text editor. -The default inventory file (-H) is ~/.ansible_hosts and is a list +The default inventory file (-H) is /etc/ansible/hosts and is a list of all hostnames to target with ansible, one per line. These can be hostnames or IPs @@ -71,8 +80,8 @@ The API is simple and returns basic datastructures. import ansible runner = ansible.Runner( pattern='*', - module_name='inventory', - host_list=['xyz.example.com', '...'] + module_name='inventory', + module_args='...' ) data = runner.run() @@ -83,15 +92,15 @@ The API is simple and returns basic datastructures. } Additional options to Runner include the number of forks, hostname -exclusion pattern, library path, arguments, and so on. Read the source, it's not -complicated. +exclusion pattern, library path, arguments, and so on. +Read the source, it's not complicated. Patterns ======== To target only hosts starting with "rtp", for example: - * ansible "rtp*" -n command -a "yum update apache" + * ansible -p "rtp*" -n command -a "yum update apache" Parallelism =========== @@ -107,19 +116,21 @@ File Transfer Ansible can SCP lots of files to lots of places in parallel. - * ansible -f 10 -n copy -a "/etc/hosts /tmp/hosts" + * ansible -p "web-*.acme.net" -f 10 -n copy -a "/etc/hosts /tmp/hosts" -Bundled Modules -=============== +Ansible Library (Bundled Modules) +================================= See the example library for modules, they can be written in any language and simply return JSON to stdout. The path to your ansible library is specified with the "-L" flag should you wish to use a different location -than "~/ansible". There is potential for a sizeable community to build +than "/usr/share/ansible". This means anyone can use Ansible, even without +root permissions. + +There is potential for a sizeable community to build up around the library scripts. -Existing library modules -======================== +Modules include: * command -- runs commands, giving output, return codes, and run time info * ping - just returns if the system is up or not @@ -129,16 +140,7 @@ Existing library modules Future plans ============ - * modules for users, groups, and files, using puppet style ensure mechanics - * ansible-inventory -- gathering fact/hw info, storing in git, adding RSS - * ansible-slurp ------ recursively rsync file trees for each host - * very simple option constructing/parsing for modules - * Dead-simple declarative configuration management engine using - a runbook style recipe file, written in JSON or YAML - * maybe it's own fact engine, not required, that also feeds from facter - * add/remove/list hosts from the command line - * list available modules from command line - * filter exclusion (run this only if fact is true/false) + * see TODO.md License ======= @@ -148,8 +150,8 @@ License Author ====== - Michael DeHaan -- michael.dehaan@gmail.com +Michael DeHaan -- michael.dehaan@gmail.com - [http://michaeldehaan.net](http://michaeldehaan.net/) +[http://michaeldehaan.net](http://michaeldehaan.net/) diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000000..9dcc533c01 --- /dev/null +++ b/TODO.md @@ -0,0 +1,15 @@ +TODO list and plans +=================== + + * make remote user settable versus assuming remote login is named root + * modules for users, groups, and files, using puppet style ensure mechanics + * ansible-inventory -- gathering fact/hw info, storing in git, adding RSS + * ansible-slurp ------ recursively rsync file trees for each host + * very simple option constructing/parsing for modules + * Dead-simple declarative configuration management engine using + a runbook style recipe file, written in JSON or YAML + * maybe it's own fact engine, not required, that also feeds from facter + * add/remove/list hosts from the command line + * list available modules from command line + * filter exclusion (run this only if fact is true/false) + From 7eb2dd2deeb7aca2292654255bf03372ff61ce27 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:04:09 -0500 Subject: [PATCH 05/33] Add remote setting to file, update TODO --- TODO.md | 5 ++--- bin/ansible | 6 +++++- lib/ansible/__init__.py | 37 ++++++++++++++++++++++--------------- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/TODO.md b/TODO.md index 9dcc533c01..07e7d79eae 100644 --- a/TODO.md +++ b/TODO.md @@ -1,10 +1,9 @@ TODO list and plans =================== - * make remote user settable versus assuming remote login is named root * modules for users, groups, and files, using puppet style ensure mechanics - * ansible-inventory -- gathering fact/hw info, storing in git, adding RSS - * ansible-slurp ------ recursively rsync file trees for each host + * ansible-inventory - gathering fact/hw info, storing in git, adding RSS + * ansible-slurp - recursively rsync file trees for each host * very simple option constructing/parsing for modules * Dead-simple declarative configuration management engine using a runbook style recipe file, written in JSON or YAML diff --git a/bin/ansible b/bin/ansible index 6425dcd714..bb8cce7295 100755 --- a/bin/ansible +++ b/bin/ansible @@ -32,6 +32,7 @@ DEFAULT_MODULE_NAME = 'ping' DEFAULT_PATTERN = '*' DEFAULT_FORKS = 3 DEFAULT_MODULE_ARGS = '' +DEFAULT_REMOTE_USER = 'root' class Cli(object): @@ -44,7 +45,7 @@ class Cli(object): help="path to hosts list", default=DEFAULT_HOST_LIST) parser.add_option("-L", "--library", dest="module_path", help="path to module library", default=DEFAULT_MODULE_PATH) - parser.add_option("-F", "--forks", dest="forks", + parser.add_option("-f", "--forks", dest="forks", help="level of parallelism", default=DEFAULT_FORKS) parser.add_option("-n", "--name", dest="module_name", help="module name to execute", default=DEFAULT_MODULE_NAME) @@ -52,6 +53,8 @@ class Cli(object): help="module arguments", default=DEFAULT_MODULE_ARGS) parser.add_option("-p", "--pattern", dest="pattern", help="hostname pattern", default=DEFAULT_PATTERN) + parser.add_option("-u", "--remote-user", dest="remote_user", + help="remote username", default=DEFAULT_REMOTE_USER) options, args = parser.parse_args() @@ -62,6 +65,7 @@ class Cli(object): module_name=options.module_name, module_path=options.module_path, module_args=options.module_args.split(' '), + remote_user=options.remote_user, host_list=options.host_list, forks=options.forks, pattern=options.pattern, diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index 4af0ba405a..77012dda9a 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -36,6 +36,7 @@ DEFAULT_PATTERN = '*' DEFAULT_FORKS = 3 DEFAULT_MODULE_ARGS = '' DEFAULT_TIMEOUT = 60 +DEFAULT_REMOTE_USER = 'root' class Pooler(object): @@ -66,6 +67,7 @@ class Runner(object): forks=DEFAULT_FORKS, timeout=DEFAULT_TIMEOUT, pattern=DEFAULT_PATTERN, + remote_user=DEFAULT_REMOTE_USER, verbose=False): @@ -81,6 +83,7 @@ class Runner(object): self.module_args = module_args self.timeout = timeout self.verbose = verbose + self.remote_user = remote_user def _parse_hosts(self, host_list): ''' parse the host inventory file if not sent as an array ''' @@ -103,23 +106,19 @@ class Runner(object): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: - ssh.connect(host, username='root', + ssh.connect(host, username=self.remote_user, allow_agent=True, look_for_keys=True) - return ssh + return [ True, ssh ] except: - # TODO -- just convert traceback to string - # and return a seperate hash of failed hosts - if self.verbose: - traceback.print_exc() - return None + return [ False, traceback.format_exc() ] def _executor(self, host): ''' callback executed in parallel for each host ''' # TODO: try/catch returning none - conn = self._connect(host) - if not conn: - return [ host, None ] + ok, conn = self._connect(host) + if not ok: + return [ host, False, conn ] if self.module_name != "copy": # transfer a module, set it executable, and run it @@ -127,15 +126,14 @@ class Runner(object): self._exec_command(conn, "chmod +x %s" % outpath) cmd = self._command(outpath) result = self._exec_command(conn, cmd) - result = json.loads(result) + return [ host, True, json.loads(result) ] else: # SFTP file copy module is not really a module ftp = conn.open_sftp() ftp.put(self.module_args[0], self.module_args[1]) ftp.close() - return [ host, 1 ] + return [ host, True, 1 ] - return [ host, result ] def _command(self, outpath): ''' form up a command string ''' @@ -168,8 +166,17 @@ class Runner(object): def executor(x): return self._executor(x) results = Pooler.parmap(executor, hosts) - by_host = dict(results) - return by_host + results2 = { + "successful" : {}, + "failed" : {} + } + for x in results: + (host, is_ok, result) = x + if not is_ok: + results2["failed"][host] = result + else: + results2["successful"][host] = result + return results2 if __name__ == '__main__': From e0e98d10ceacb9853cb5e6cb7c8f78cf8db3263b Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:09:23 -0500 Subject: [PATCH 06/33] use readlines on stdout so we'll block on long running commands --- lib/ansible/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index 77012dda9a..37d6d38e51 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -25,6 +25,7 @@ from itertools import izip import os import json import traceback +import select # non-core import paramiko @@ -143,7 +144,7 @@ class Runner(object): def _exec_command(self, conn, cmd): ''' execute a command over SSH ''' stdin, stdout, stderr = conn.exec_command(cmd) - results = stdout.read() + results = "\n".join(stdout.readlines()) return results def _copy_module(self, conn): From 7ce5db97ed8fd5233b0fd7e93c34e1c3da1788f6 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:10:34 -0500 Subject: [PATCH 07/33] Add explicit calls to close connections --- lib/ansible/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index 37d6d38e51..d5c0b95c73 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -127,12 +127,14 @@ class Runner(object): self._exec_command(conn, "chmod +x %s" % outpath) cmd = self._command(outpath) result = self._exec_command(conn, cmd) + conn.close() return [ host, True, json.loads(result) ] else: # SFTP file copy module is not really a module ftp = conn.open_sftp() ftp.put(self.module_args[0], self.module_args[1]) ftp.close() + conn.close() return [ host, True, 1 ] From 4608a93de59ef274b8820dcff88af786cb73f1c8 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:24:56 -0500 Subject: [PATCH 08/33] Added docs on split success/failure structures --- bin/ansible-inventory | 80 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 bin/ansible-inventory diff --git a/bin/ansible-inventory b/bin/ansible-inventory new file mode 100755 index 0000000000..bb8cce7295 --- /dev/null +++ b/bin/ansible-inventory @@ -0,0 +1,80 @@ +#!/usr/bin/python + +# Copyright (c) 2012 Michael DeHaan +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR +# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from optparse import OptionParser +import json +import os +import ansible + +DEFAULT_HOST_LIST = '/etc/ansible/hosts' +DEFAULT_MODULE_PATH = '/usr/share/ansible' +DEFAULT_MODULE_NAME = 'ping' +DEFAULT_PATTERN = '*' +DEFAULT_FORKS = 3 +DEFAULT_MODULE_ARGS = '' +DEFAULT_REMOTE_USER = 'root' + +class Cli(object): + + def __init__(self): + pass + + def runner(self): + parser = OptionParser() + parser.add_option("-H", "--host-list", dest="host_list", + help="path to hosts list", default=DEFAULT_HOST_LIST) + parser.add_option("-L", "--library", dest="module_path", + help="path to module library", default=DEFAULT_MODULE_PATH) + parser.add_option("-f", "--forks", dest="forks", + help="level of parallelism", default=DEFAULT_FORKS) + parser.add_option("-n", "--name", dest="module_name", + help="module name to execute", default=DEFAULT_MODULE_NAME) + parser.add_option("-a", "--args", dest="module_args", + help="module arguments", default=DEFAULT_MODULE_ARGS) + parser.add_option("-p", "--pattern", dest="pattern", + help="hostname pattern", default=DEFAULT_PATTERN) + parser.add_option("-u", "--remote-user", dest="remote_user", + help="remote username", default=DEFAULT_REMOTE_USER) + + options, args = parser.parse_args() + + # TODO: more shell like splitting on module_args would + # be a good idea + + return ansible.Runner( + module_name=options.module_name, + module_path=options.module_path, + module_args=options.module_args.split(' '), + remote_user=options.remote_user, + host_list=options.host_list, + forks=options.forks, + pattern=options.pattern, + verbose=False, + ) + +if __name__ == '__main__': + + result = Cli().runner().run() + print json.dumps(result, sort_keys=True, indent=4) + + From 530e54b3e44becd9ad631a6500e4232ccadac1af Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:38:51 -0500 Subject: [PATCH 09/33] Fix multiprocessing pool usage and remove stackoverflow hack --- README.md | 12 ++++--- bin/ansible-inventory | 80 ----------------------------------------- lib/ansible/__init__.py | 29 ++++----------- 3 files changed, 15 insertions(+), 106 deletions(-) delete mode 100755 bin/ansible-inventory diff --git a/README.md b/README.md index 71fc7f5015..e58f194651 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,14 @@ The API is simple and returns basic datastructures. ) data = runner.run() - { - 'xyz.example.com' : [ 'any kind of datastructure is returnable' ], - 'foo.example.com' : None, # failed to connect, - ... + { + 'successful' : { + 'xyz.example.com' : [ 'any kind of datastructure is returnable' ], + 'foo.example.com' : [ '...' ] + }, + 'failed' : { + 'bar.example.com' : [ 'failure message' ] + } } Additional options to Runner include the number of forks, hostname diff --git a/bin/ansible-inventory b/bin/ansible-inventory deleted file mode 100755 index bb8cce7295..0000000000 --- a/bin/ansible-inventory +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2012 Michael DeHaan -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR -# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from optparse import OptionParser -import json -import os -import ansible - -DEFAULT_HOST_LIST = '/etc/ansible/hosts' -DEFAULT_MODULE_PATH = '/usr/share/ansible' -DEFAULT_MODULE_NAME = 'ping' -DEFAULT_PATTERN = '*' -DEFAULT_FORKS = 3 -DEFAULT_MODULE_ARGS = '' -DEFAULT_REMOTE_USER = 'root' - -class Cli(object): - - def __init__(self): - pass - - def runner(self): - parser = OptionParser() - parser.add_option("-H", "--host-list", dest="host_list", - help="path to hosts list", default=DEFAULT_HOST_LIST) - parser.add_option("-L", "--library", dest="module_path", - help="path to module library", default=DEFAULT_MODULE_PATH) - parser.add_option("-f", "--forks", dest="forks", - help="level of parallelism", default=DEFAULT_FORKS) - parser.add_option("-n", "--name", dest="module_name", - help="module name to execute", default=DEFAULT_MODULE_NAME) - parser.add_option("-a", "--args", dest="module_args", - help="module arguments", default=DEFAULT_MODULE_ARGS) - parser.add_option("-p", "--pattern", dest="pattern", - help="hostname pattern", default=DEFAULT_PATTERN) - parser.add_option("-u", "--remote-user", dest="remote_user", - help="remote username", default=DEFAULT_REMOTE_USER) - - options, args = parser.parse_args() - - # TODO: more shell like splitting on module_args would - # be a good idea - - return ansible.Runner( - module_name=options.module_name, - module_path=options.module_path, - module_args=options.module_args.split(' '), - remote_user=options.remote_user, - host_list=options.host_list, - forks=options.forks, - pattern=options.pattern, - verbose=False, - ) - -if __name__ == '__main__': - - result = Cli().runner().run() - print json.dumps(result, sort_keys=True, indent=4) - - diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index d5c0b95c73..420b3fe039 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -20,7 +20,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import fnmatch -from multiprocessing import Process, Pipe +import multiprocessing from itertools import izip import os import json @@ -39,24 +39,9 @@ DEFAULT_MODULE_ARGS = '' DEFAULT_TIMEOUT = 60 DEFAULT_REMOTE_USER = 'root' -class Pooler(object): - - # credit: http://stackoverflow.com/questions/3288595/multiprocessing-using-pool-map-on-a-function-defined-in-a-class - - @classmethod - def spawn(cls, f): - def fun(pipe,x): - pipe.send(f(x)) - pipe.close() - return fun - - @classmethod - def parmap(cls, f, X): - pipe=[Pipe() for x in X] - proc=[Process(target=cls.spawn(f),args=(c,x)) for x,(p,c) in izip(X,pipe)] - [p.start() for p in proc] - [p.join() for p in proc] - return [p.recv() for (p,c) in pipe] +def _executor_hook(x): + (runner, host) = x + return runner._executor(host) class Runner(object): @@ -166,9 +151,9 @@ class Runner(object): def run(self): ''' xfer & run module on all matched hosts ''' hosts = [ h for h in self.host_list if self._matches(h) ] - def executor(x): - return self._executor(x) - results = Pooler.parmap(executor, hosts) + pool = multiprocessing.Pool(self.forks) + hosts = [ (self,x) for x in hosts ] + results = pool.map(_executor_hook, hosts) results2 = { "successful" : {}, "failed" : {} From 7be8d134c9f91d71c08270786a0ca18a2e49b1e9 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:47:03 -0500 Subject: [PATCH 10/33] Rename 'successful' to 'contacted' ... --- README.md | 10 +++++++--- lib/ansible/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e58f194651..b88d6e88e5 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,10 @@ Run a module by name with arguments API Example =========== -The API is simple and returns basic datastructures. +The API is simple and returns basic datastructures. Ansible will keep +track of which hosts were successfully contacted seperately from hosts +that had communication problems. The format of the return, if successful, +is entirely up to the module. import ansible runner = ansible.Runner( @@ -86,17 +89,18 @@ The API is simple and returns basic datastructures. data = runner.run() { - 'successful' : { + 'contacted' : { 'xyz.example.com' : [ 'any kind of datastructure is returnable' ], 'foo.example.com' : [ '...' ] }, - 'failed' : { + 'dark' : { 'bar.example.com' : [ 'failure message' ] } } Additional options to Runner include the number of forks, hostname exclusion pattern, library path, arguments, and so on. + Read the source, it's not complicated. Patterns diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index 420b3fe039..39375e4d55 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -155,8 +155,8 @@ class Runner(object): hosts = [ (self,x) for x in hosts ] results = pool.map(_executor_hook, hosts) results2 = { - "successful" : {}, - "failed" : {} + "contacted" : {}, + "dark" : {} } for x in results: (host, is_ok, result) = x From 659c0efd722acc6e8d4b5218594c7954f9cb77f0 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:50:25 -0500 Subject: [PATCH 11/33] Add authors file to list contributors --- AUTHORS.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 AUTHORS.md diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000000..2fbe5e56ea --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,10 @@ +Patches and Contributions +========================= + + * Michael DeHaan - michael.dehaan AT gmail DOT com + * Jeremy Katz - katzj AT fedoraproject DOT org + +Send in a github pull request to get your name here. + +Upstream: github.com/mpdehaan/ansible + From 11f79300385b40cc19106837a75dc412e9693857 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 22:54:01 -0500 Subject: [PATCH 12/33] trim unused modules --- lib/ansible/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index 39375e4d55..c740f88ae0 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -21,11 +21,9 @@ import fnmatch import multiprocessing -from itertools import izip import os import json import traceback -import select # non-core import paramiko From bd37864242907f780162e7af87506a5f450579bc Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:00:37 -0500 Subject: [PATCH 13/33] Comments and fixup on the dark/contacted code --- lib/ansible/__init__.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index c740f88ae0..b57aee7aaa 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -38,6 +38,7 @@ DEFAULT_TIMEOUT = 60 DEFAULT_REMOTE_USER = 'root' def _executor_hook(x): + ''' callback used by multiprocessing pool ''' (runner, host) = x return runner._executor(host) @@ -86,7 +87,11 @@ class Runner(object): return False def _connect(self, host): - ''' obtains a paramiko connection to the host ''' + ''' + obtains a paramiko connection to the host. + on success, returns (True, connection) + on failure, returns (False, traceback str) + ''' ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: @@ -97,8 +102,13 @@ class Runner(object): return [ False, traceback.format_exc() ] def _executor(self, host): - ''' callback executed in parallel for each host ''' - # TODO: try/catch returning none + ''' + callback executed in parallel for each host. + returns (hostname, connected_ok, extra) + where extra is the result of a successful connect + or a traceback string + ''' + # TODO: try/catch around JSON handling ok, conn = self._connect(host) if not ok: @@ -148,10 +158,17 @@ class Runner(object): def run(self): ''' xfer & run module on all matched hosts ''' + + # find hosts that match the pattern hosts = [ h for h in self.host_list if self._matches(h) ] + + # attack pool of hosts in N forks pool = multiprocessing.Pool(self.forks) hosts = [ (self,x) for x in hosts ] results = pool.map(_executor_hook, hosts) + + # sort hosts by ones we successfully contacted + # and ones we did not results2 = { "contacted" : {}, "dark" : {} @@ -159,9 +176,10 @@ class Runner(object): for x in results: (host, is_ok, result) = x if not is_ok: - results2["failed"][host] = result + results2["dark"][host] = result else: - results2["successful"][host] = result + results2["contacted"][host] = result + return results2 From 25df80ff58aae6ada66e5ca2e9fd21eb106dbe80 Mon Sep 17 00:00:00 2001 From: Jeremy Katz Date: Thu, 23 Feb 2012 21:01:02 -0500 Subject: [PATCH 14/33] Use a mktemp'd path for uploading modules --- lib/ansible/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index b57aee7aaa..374e7e9944 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -120,6 +120,7 @@ class Runner(object): self._exec_command(conn, "chmod +x %s" % outpath) cmd = self._command(outpath) result = self._exec_command(conn, cmd) + self._exec_command(conn, "rm -f %s" % outpath) conn.close() return [ host, True, json.loads(result) ] else: @@ -142,15 +143,17 @@ class Runner(object): results = "\n".join(stdout.readlines()) return results + def _get_tmp_path(self, conn, file_name): + output = self._exec_command(conn, "mktemp /tmp/%s.XXXXXX" % file_name) + return output.split("\n")[0] + def _copy_module(self, conn): ''' transfer a module over SFTP ''' in_path = os.path.expanduser( os.path.join(self.module_path, self.module_name) ) - out_path = os.path.join( - "/var/spool/", - "ansible_%s" % self.module_name - ) + out_path = self._get_tmp_path(conn, "ansible_%s" % self.module_name) + sftp = conn.open_sftp() sftp.put(in_path, out_path) sftp.close() From 24e10dc2e8df8caf2dad33f4cc0776742a1afeba Mon Sep 17 00:00:00 2001 From: Jeremy Katz Date: Fri, 24 Feb 2012 16:10:53 -0500 Subject: [PATCH 15/33] Don't use a shell and thus avoid a whole class of problems --- library/command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/command b/library/command index c11cfebd25..780e50dd6d 100755 --- a/library/command +++ b/library/command @@ -11,7 +11,7 @@ import datetime args = sys.argv[1:] startd = datetime.datetime.now() -cmd = subprocess.Popen(args, shell=True, +cmd = subprocess.Popen(args, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() From 08b45d6da1d546fa64db076da58c56ea496685ae Mon Sep 17 00:00:00 2001 From: Seth Vidal Date: Fri, 24 Feb 2012 18:13:11 -0500 Subject: [PATCH 16/33] add support to prompt for ssh password on the cli --- bin/ansible | 8 ++++++++ lib/ansible/__init__.py | 7 +++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/ansible b/bin/ansible index bb8cce7295..690fcb6dea 100755 --- a/bin/ansible +++ b/bin/ansible @@ -24,6 +24,7 @@ from optparse import OptionParser import json import os +import getpass import ansible DEFAULT_HOST_LIST = '/etc/ansible/hosts' @@ -41,6 +42,8 @@ class Cli(object): def runner(self): parser = OptionParser() + parser.add_option("-P", "--askpass", default=False, action="store_true", + help="ask the user to input the ssh password for connecting") parser.add_option("-H", "--host-list", dest="host_list", help="path to hosts list", default=DEFAULT_HOST_LIST) parser.add_option("-L", "--library", dest="module_path", @@ -61,11 +64,16 @@ class Cli(object): # TODO: more shell like splitting on module_args would # be a good idea + sshpass = None + if options.askpass: + sshpass = getpass.getpass(prompt="SSH password: ") + return ansible.Runner( module_name=options.module_name, module_path=options.module_path, module_args=options.module_args.split(' '), remote_user=options.remote_user, + remote_pass=sshpass, host_list=options.host_list, forks=options.forks, pattern=options.pattern, diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index b57aee7aaa..f4f3d9a4b2 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -36,6 +36,7 @@ DEFAULT_FORKS = 3 DEFAULT_MODULE_ARGS = '' DEFAULT_TIMEOUT = 60 DEFAULT_REMOTE_USER = 'root' +DEFAULT_REMOTE_PASS = None def _executor_hook(x): ''' callback used by multiprocessing pool ''' @@ -53,6 +54,7 @@ class Runner(object): timeout=DEFAULT_TIMEOUT, pattern=DEFAULT_PATTERN, remote_user=DEFAULT_REMOTE_USER, + remote_pass=DEFAULT_REMOTE_PASS, verbose=False): @@ -69,6 +71,7 @@ class Runner(object): self.timeout = timeout self.verbose = verbose self.remote_user = remote_user + self.remote_pass = remote_pass def _parse_hosts(self, host_list): ''' parse the host inventory file if not sent as an array ''' @@ -95,8 +98,8 @@ class Runner(object): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: - ssh.connect(host, username=self.remote_user, - allow_agent=True, look_for_keys=True) + ssh.connect(host, username=self.remote_user, allow_agent=True, + look_for_keys=True, password=self.remote_pass) return [ True, ssh ] except: return [ False, traceback.format_exc() ] From be9fdc8ef12e7a4c4db32fa7889fe0fe93d17f8d Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:03:57 -0500 Subject: [PATCH 17/33] Add Seth to authors file. --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 2fbe5e56ea..5d506b4cd3 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -3,6 +3,7 @@ Patches and Contributions * Michael DeHaan - michael.dehaan AT gmail DOT com * Jeremy Katz - katzj AT fedoraproject DOT org + * Seth Vidal - skvidal AT fedoraproject DOT org Send in a github pull request to get your name here. From 6eda2cf383040271e191b883599043eddd1d6cc3 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:26:16 -0500 Subject: [PATCH 18/33] Added initial stub for where playbooks will go, moved to common constants file so as to not repeat constants between CLI and lib. --- README.md | 10 ++++++ bin/ansible | 58 +++++++++++++++++-------------- lib/ansible/__init__.py | 28 ++++++--------- lib/ansible/constants.py | 10 ++++++ lib/ansible/playbook.py | 74 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 44 deletions(-) create mode 100644 lib/ansible/constants.py create mode 100755 lib/ansible/playbook.py diff --git a/README.md b/README.md index b88d6e88e5..1b6e919e23 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Requirements For the server the tool is running from, *only*: * python 2.6 -- or the 2.4/2.5 backport of the multiprocessing module + * PyYAML (if using playbooks) * paramiko Inventory file @@ -145,6 +146,15 @@ Modules include: * facter - retrieves facts about the host OS * copy - add files to remote systems +Playbooks +========= + +Playbooks are loosely equivalent to recipes or manifests in most configuration +management or deployment tools and describe a set of operations to run on +a set of hosts. Some tasks can choose to only fire when certain +conditions are true, and if a task in a chain fails the dependent tasks +will not proceed. Playbooks are described in (YAML)[http://yaml.org] format. + Future plans ============ diff --git a/bin/ansible b/bin/ansible index 690fcb6dea..94ff3921a5 100755 --- a/bin/ansible +++ b/bin/ansible @@ -26,14 +26,8 @@ import json import os import getpass import ansible - -DEFAULT_HOST_LIST = '/etc/ansible/hosts' -DEFAULT_MODULE_PATH = '/usr/share/ansible' -DEFAULT_MODULE_NAME = 'ping' -DEFAULT_PATTERN = '*' -DEFAULT_FORKS = 3 -DEFAULT_MODULE_ARGS = '' -DEFAULT_REMOTE_USER = 'root' +import ansible.playbook +import ansible.constants as C class Cli(object): @@ -45,19 +39,21 @@ class Cli(object): parser.add_option("-P", "--askpass", default=False, action="store_true", help="ask the user to input the ssh password for connecting") parser.add_option("-H", "--host-list", dest="host_list", - help="path to hosts list", default=DEFAULT_HOST_LIST) + help="path to hosts list", default=C.DEFAULT_HOST_LIST) parser.add_option("-L", "--library", dest="module_path", - help="path to module library", default=DEFAULT_MODULE_PATH) + help="path to module library", default=C.DEFAULT_MODULE_PATH) parser.add_option("-f", "--forks", dest="forks", - help="level of parallelism", default=DEFAULT_FORKS) + help="level of parallelism", default=C.DEFAULT_FORKS) parser.add_option("-n", "--name", dest="module_name", - help="module name to execute", default=DEFAULT_MODULE_NAME) + help="module name to execute", default=C.DEFAULT_MODULE_NAME) parser.add_option("-a", "--args", dest="module_args", - help="module arguments", default=DEFAULT_MODULE_ARGS) + help="module arguments", default=C.DEFAULT_MODULE_ARGS) parser.add_option("-p", "--pattern", dest="pattern", - help="hostname pattern", default=DEFAULT_PATTERN) + help="hostname pattern", default=C.DEFAULT_PATTERN) parser.add_option("-u", "--remote-user", dest="remote_user", - help="remote username", default=DEFAULT_REMOTE_USER) + help="remote username", default=C.DEFAULT_REMOTE_USER) + parser.add_option("-r", "--run-playbook", dest="playbook", + help="playbook file, instead of -n and -a", default=None) options, args = parser.parse_args() @@ -68,17 +64,27 @@ class Cli(object): if options.askpass: sshpass = getpass.getpass(prompt="SSH password: ") - return ansible.Runner( - module_name=options.module_name, - module_path=options.module_path, - module_args=options.module_args.split(' '), - remote_user=options.remote_user, - remote_pass=sshpass, - host_list=options.host_list, - forks=options.forks, - pattern=options.pattern, - verbose=False, - ) + if options.playbook is None: + return ansible.Runner( + module_name=options.module_name, + module_path=options.module_path, + module_args=options.module_args.split(' '), + remote_user=options.remote_user, + remote_pass=sshpass, + host_list=options.host_list, + forks=options.forks, + pattern=options.pattern, + verbose=False, + ) + else: + return ansible.playbook.PlayBook( + module_path=options.module_path, + remote_user=options.remote_user, + remote_pass=sshpass, + host_list=options.host_list, + forks=options.forks, + verbose=False, + ) if __name__ == '__main__': diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index f58797061c..2517e51455 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -28,15 +28,7 @@ import traceback # non-core import paramiko -DEFAULT_HOST_LIST = '/etc/ansible/hosts' -DEFAULT_MODULE_PATH = '/usr/share/ansible' -DEFAULT_MODULE_NAME = 'ping' -DEFAULT_PATTERN = '*' -DEFAULT_FORKS = 3 -DEFAULT_MODULE_ARGS = '' -DEFAULT_TIMEOUT = 60 -DEFAULT_REMOTE_USER = 'root' -DEFAULT_REMOTE_PASS = None +import constants as C def _executor_hook(x): ''' callback used by multiprocessing pool ''' @@ -46,15 +38,15 @@ def _executor_hook(x): class Runner(object): def __init__(self, - host_list=DEFAULT_HOST_LIST, - module_path=DEFAULT_MODULE_PATH, - module_name=DEFAULT_MODULE_NAME, - module_args=DEFAULT_MODULE_ARGS, - forks=DEFAULT_FORKS, - timeout=DEFAULT_TIMEOUT, - pattern=DEFAULT_PATTERN, - remote_user=DEFAULT_REMOTE_USER, - remote_pass=DEFAULT_REMOTE_PASS, + host_list=C.DEFAULT_HOST_LIST, + module_path=C.DEFAULT_MODULE_PATH, + module_name=C.DEFAULT_MODULE_NAME, + module_args=C.DEFAULT_MODULE_ARGS, + forks=C.DEFAULT_FORKS, + timeout=C.DEFAULT_TIMEOUT, + pattern=C.DEFAULT_PATTERN, + remote_user=C.DEFAULT_REMOTE_USER, + remote_pass=C.DEFAULT_REMOTE_PASS, verbose=False): diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py new file mode 100644 index 0000000000..15ad18de6b --- /dev/null +++ b/lib/ansible/constants.py @@ -0,0 +1,10 @@ +DEFAULT_HOST_LIST = '/etc/ansible/hosts' +DEFAULT_MODULE_PATH = '/usr/share/ansible' +DEFAULT_MODULE_NAME = 'ping' +DEFAULT_PATTERN = '*' +DEFAULT_FORKS = 3 +DEFAULT_MODULE_ARGS = '' +DEFAULT_TIMEOUT = 60 +DEFAULT_REMOTE_USER = 'root' +DEFAULT_REMOTE_PASS = None + diff --git a/lib/ansible/playbook.py b/lib/ansible/playbook.py new file mode 100755 index 0000000000..d88946d4e1 --- /dev/null +++ b/lib/ansible/playbook.py @@ -0,0 +1,74 @@ +# Copyright (c) 2012 Michael DeHaan +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR +# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import ansible +import ansible.constants as C +import json +import yaml + +# TODO: make a constants file rather than +# duplicating these + +class PlayBook(object): + ''' + runs an ansible playbook, given as a datastructure + or YAML filename + ''' + + def __init__(self, + playbook =None, + host_list =C.DEFAULT_HOST_LIST, + module_path =C.DEFAULT_MODULE_PATH, + forks =C.DEFAULT_FORKS, + timeout =C.DEFAULT_TIMEOUT, + remote_user =C.DEFAULT_REMOTE_USER, + remote_pass =C.DEFAULT_REMOTE_PASS, + verbose=False): + + # runner is reused between calls + + self.runner = ansible.Runner( + host_list=host_list, + module_path=module_path, + forks=forks, + timeout=timeout, + remote_user=remote_user, + remote_pass=remote_pass, + verbose=verbose + ) + + if type(playbook) == str: + playbook = yaml.load(file(playbook).read()) + + def run(self): + pass + +# r = Runner( +# host_list = DEFAULT_HOST_LIST, +# module_name='ping', +# module_args='', +# pattern='*', +# forks=3 +# ) +# print r.run() + + + From d079c8e5f3f8dce3ae1814fd797e53cb7354a43c Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:28:58 -0500 Subject: [PATCH 19/33] Move runner out of __init__.py so it's clear what classes live where. --- bin/ansible | 4 +- lib/ansible/__init__.py | 198 ---------------------------------------- lib/ansible/playbook.py | 4 +- lib/ansible/runner.py | 198 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 202 deletions(-) create mode 100755 lib/ansible/runner.py diff --git a/bin/ansible b/bin/ansible index 94ff3921a5..4224a6efc9 100755 --- a/bin/ansible +++ b/bin/ansible @@ -25,7 +25,7 @@ from optparse import OptionParser import json import os import getpass -import ansible +import ansible.runner import ansible.playbook import ansible.constants as C @@ -65,7 +65,7 @@ class Cli(object): sshpass = getpass.getpass(prompt="SSH password: ") if options.playbook is None: - return ansible.Runner( + return ansible.runner.Runner( module_name=options.module_name, module_path=options.module_path, module_args=options.module_args.split(' '), diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index 2517e51455..e69de29bb2 100755 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -1,198 +0,0 @@ -# Copyright (c) 2012 Michael DeHaan -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR -# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import fnmatch -import multiprocessing -import os -import json -import traceback - -# non-core -import paramiko - -import constants as C - -def _executor_hook(x): - ''' callback used by multiprocessing pool ''' - (runner, host) = x - return runner._executor(host) - -class Runner(object): - - def __init__(self, - host_list=C.DEFAULT_HOST_LIST, - module_path=C.DEFAULT_MODULE_PATH, - module_name=C.DEFAULT_MODULE_NAME, - module_args=C.DEFAULT_MODULE_ARGS, - forks=C.DEFAULT_FORKS, - timeout=C.DEFAULT_TIMEOUT, - pattern=C.DEFAULT_PATTERN, - remote_user=C.DEFAULT_REMOTE_USER, - remote_pass=C.DEFAULT_REMOTE_PASS, - verbose=False): - - - ''' - Constructor. - ''' - - self.host_list = self._parse_hosts(host_list) - self.module_path = module_path - self.module_name = module_name - self.forks = forks - self.pattern = pattern - self.module_args = module_args - self.timeout = timeout - self.verbose = verbose - self.remote_user = remote_user - self.remote_pass = remote_pass - - def _parse_hosts(self, host_list): - ''' parse the host inventory file if not sent as an array ''' - if type(host_list) != list: - host_list = os.path.expanduser(host_list) - return file(host_list).read().split("\n") - return host_list - - - def _matches(self, host_name): - ''' returns if a hostname is matched by the pattern ''' - if host_name == '': - return False - if fnmatch.fnmatch(host_name, self.pattern): - return True - return False - - def _connect(self, host): - ''' - obtains a paramiko connection to the host. - on success, returns (True, connection) - on failure, returns (False, traceback str) - ''' - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - try: - ssh.connect(host, username=self.remote_user, allow_agent=True, - look_for_keys=True, password=self.remote_pass) - return [ True, ssh ] - except: - return [ False, traceback.format_exc() ] - - def _executor(self, host): - ''' - callback executed in parallel for each host. - returns (hostname, connected_ok, extra) - where extra is the result of a successful connect - or a traceback string - ''' - # TODO: try/catch around JSON handling - - ok, conn = self._connect(host) - if not ok: - return [ host, False, conn ] - - if self.module_name != "copy": - # transfer a module, set it executable, and run it - outpath = self._copy_module(conn) - self._exec_command(conn, "chmod +x %s" % outpath) - cmd = self._command(outpath) - result = self._exec_command(conn, cmd) - self._exec_command(conn, "rm -f %s" % outpath) - conn.close() - return [ host, True, json.loads(result) ] - else: - # SFTP file copy module is not really a module - ftp = conn.open_sftp() - ftp.put(self.module_args[0], self.module_args[1]) - ftp.close() - conn.close() - return [ host, True, 1 ] - - - def _command(self, outpath): - ''' form up a command string ''' - cmd = "%s %s" % (outpath, " ".join(self.module_args)) - return cmd - - def _exec_command(self, conn, cmd): - ''' execute a command over SSH ''' - stdin, stdout, stderr = conn.exec_command(cmd) - results = "\n".join(stdout.readlines()) - return results - - def _get_tmp_path(self, conn, file_name): - output = self._exec_command(conn, "mktemp /tmp/%s.XXXXXX" % file_name) - return output.split("\n")[0] - - def _copy_module(self, conn): - ''' transfer a module over SFTP ''' - in_path = os.path.expanduser( - os.path.join(self.module_path, self.module_name) - ) - out_path = self._get_tmp_path(conn, "ansible_%s" % self.module_name) - - sftp = conn.open_sftp() - sftp.put(in_path, out_path) - sftp.close() - return out_path - - def run(self): - ''' xfer & run module on all matched hosts ''' - - # find hosts that match the pattern - hosts = [ h for h in self.host_list if self._matches(h) ] - - # attack pool of hosts in N forks - pool = multiprocessing.Pool(self.forks) - hosts = [ (self,x) for x in hosts ] - results = pool.map(_executor_hook, hosts) - - # sort hosts by ones we successfully contacted - # and ones we did not - results2 = { - "contacted" : {}, - "dark" : {} - } - for x in results: - (host, is_ok, result) = x - if not is_ok: - results2["dark"][host] = result - else: - results2["contacted"][host] = result - - return results2 - - -if __name__ == '__main__': - - # test code... - - r = Runner( - host_list = DEFAULT_HOST_LIST, - module_name='ping', - module_args='', - pattern='*', - forks=3 - ) - print r.run() - - - diff --git a/lib/ansible/playbook.py b/lib/ansible/playbook.py index d88946d4e1..fa6c27731e 100755 --- a/lib/ansible/playbook.py +++ b/lib/ansible/playbook.py @@ -19,7 +19,7 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import ansible +import ansible.runner import ansible.constants as C import json import yaml @@ -45,7 +45,7 @@ class PlayBook(object): # runner is reused between calls - self.runner = ansible.Runner( + self.runner = ansible.runner.Runner( host_list=host_list, module_path=module_path, forks=forks, diff --git a/lib/ansible/runner.py b/lib/ansible/runner.py new file mode 100755 index 0000000000..2517e51455 --- /dev/null +++ b/lib/ansible/runner.py @@ -0,0 +1,198 @@ +# Copyright (c) 2012 Michael DeHaan +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR +# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch +import multiprocessing +import os +import json +import traceback + +# non-core +import paramiko + +import constants as C + +def _executor_hook(x): + ''' callback used by multiprocessing pool ''' + (runner, host) = x + return runner._executor(host) + +class Runner(object): + + def __init__(self, + host_list=C.DEFAULT_HOST_LIST, + module_path=C.DEFAULT_MODULE_PATH, + module_name=C.DEFAULT_MODULE_NAME, + module_args=C.DEFAULT_MODULE_ARGS, + forks=C.DEFAULT_FORKS, + timeout=C.DEFAULT_TIMEOUT, + pattern=C.DEFAULT_PATTERN, + remote_user=C.DEFAULT_REMOTE_USER, + remote_pass=C.DEFAULT_REMOTE_PASS, + verbose=False): + + + ''' + Constructor. + ''' + + self.host_list = self._parse_hosts(host_list) + self.module_path = module_path + self.module_name = module_name + self.forks = forks + self.pattern = pattern + self.module_args = module_args + self.timeout = timeout + self.verbose = verbose + self.remote_user = remote_user + self.remote_pass = remote_pass + + def _parse_hosts(self, host_list): + ''' parse the host inventory file if not sent as an array ''' + if type(host_list) != list: + host_list = os.path.expanduser(host_list) + return file(host_list).read().split("\n") + return host_list + + + def _matches(self, host_name): + ''' returns if a hostname is matched by the pattern ''' + if host_name == '': + return False + if fnmatch.fnmatch(host_name, self.pattern): + return True + return False + + def _connect(self, host): + ''' + obtains a paramiko connection to the host. + on success, returns (True, connection) + on failure, returns (False, traceback str) + ''' + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + ssh.connect(host, username=self.remote_user, allow_agent=True, + look_for_keys=True, password=self.remote_pass) + return [ True, ssh ] + except: + return [ False, traceback.format_exc() ] + + def _executor(self, host): + ''' + callback executed in parallel for each host. + returns (hostname, connected_ok, extra) + where extra is the result of a successful connect + or a traceback string + ''' + # TODO: try/catch around JSON handling + + ok, conn = self._connect(host) + if not ok: + return [ host, False, conn ] + + if self.module_name != "copy": + # transfer a module, set it executable, and run it + outpath = self._copy_module(conn) + self._exec_command(conn, "chmod +x %s" % outpath) + cmd = self._command(outpath) + result = self._exec_command(conn, cmd) + self._exec_command(conn, "rm -f %s" % outpath) + conn.close() + return [ host, True, json.loads(result) ] + else: + # SFTP file copy module is not really a module + ftp = conn.open_sftp() + ftp.put(self.module_args[0], self.module_args[1]) + ftp.close() + conn.close() + return [ host, True, 1 ] + + + def _command(self, outpath): + ''' form up a command string ''' + cmd = "%s %s" % (outpath, " ".join(self.module_args)) + return cmd + + def _exec_command(self, conn, cmd): + ''' execute a command over SSH ''' + stdin, stdout, stderr = conn.exec_command(cmd) + results = "\n".join(stdout.readlines()) + return results + + def _get_tmp_path(self, conn, file_name): + output = self._exec_command(conn, "mktemp /tmp/%s.XXXXXX" % file_name) + return output.split("\n")[0] + + def _copy_module(self, conn): + ''' transfer a module over SFTP ''' + in_path = os.path.expanduser( + os.path.join(self.module_path, self.module_name) + ) + out_path = self._get_tmp_path(conn, "ansible_%s" % self.module_name) + + sftp = conn.open_sftp() + sftp.put(in_path, out_path) + sftp.close() + return out_path + + def run(self): + ''' xfer & run module on all matched hosts ''' + + # find hosts that match the pattern + hosts = [ h for h in self.host_list if self._matches(h) ] + + # attack pool of hosts in N forks + pool = multiprocessing.Pool(self.forks) + hosts = [ (self,x) for x in hosts ] + results = pool.map(_executor_hook, hosts) + + # sort hosts by ones we successfully contacted + # and ones we did not + results2 = { + "contacted" : {}, + "dark" : {} + } + for x in results: + (host, is_ok, result) = x + if not is_ok: + results2["dark"][host] = result + else: + results2["contacted"][host] = result + + return results2 + + +if __name__ == '__main__': + + # test code... + + r = Runner( + host_list = DEFAULT_HOST_LIST, + module_name='ping', + module_args='', + pattern='*', + forks=3 + ) + print r.run() + + + From 0095336a4e7228c9971bdadedf17eefd06ba5002 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:29:34 -0500 Subject: [PATCH 20/33] Update API docs for runner --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b6e919e23..11309e3cf8 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ that had communication problems. The format of the return, if successful, is entirely up to the module. import ansible - runner = ansible.Runner( + runner = ansible.runner.Runner( pattern='*', module_name='inventory', module_args='...' From 7730341d2487ffc52f0bdd1b9f0dbf65e02ae1cb Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:54:16 -0500 Subject: [PATCH 21/33] We don't have modules that list when things change just yet. I plan to handle this by having a changed=True/False in the JSON for these modules. Added a note so folks won't think we can only execute shell :) --- examples/playbook.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 examples/playbook.yml diff --git a/examples/playbook.yml b/examples/playbook.yml new file mode 100644 index 0000000000..7b5ec9c5c4 --- /dev/null +++ b/examples/playbook.yml @@ -0,0 +1,16 @@ +- pattern: '*.prod.example.com' + tasks: + - do: + - update apache (note: service module TBD) + - command + - [/usr/bin/yum, update, apache] + onchange: + - do: + - restart apache (note: service module TBD) + - command + - [/sbin/service, apache, restart] + - do: + - run bin false + - command + - [/bin/false] + From 767517ac65221c14d99f3b6853dec956f915961e Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:57:26 -0500 Subject: [PATCH 22/33] Force forks to be an integer when read by command line --- bin/ansible | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ansible b/bin/ansible index 4224a6efc9..4254b0b7fb 100755 --- a/bin/ansible +++ b/bin/ansible @@ -42,7 +42,7 @@ class Cli(object): help="path to hosts list", default=C.DEFAULT_HOST_LIST) parser.add_option("-L", "--library", dest="module_path", help="path to module library", default=C.DEFAULT_MODULE_PATH) - parser.add_option("-f", "--forks", dest="forks", + parser.add_option("-f", "--forks", dest="forks", type="int", help="level of parallelism", default=C.DEFAULT_FORKS) parser.add_option("-n", "--name", dest="module_name", help="module name to execute", default=C.DEFAULT_MODULE_NAME) From 2fc109b47a1018e2b50c2a7151d41d38a26dbefd Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 23 Feb 2012 23:58:40 -0500 Subject: [PATCH 23/33] Credit Tim for last patch. --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 5d506b4cd3..eb5b5f301f 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -4,6 +4,7 @@ Patches and Contributions * Michael DeHaan - michael.dehaan AT gmail DOT com * Jeremy Katz - katzj AT fedoraproject DOT org * Seth Vidal - skvidal AT fedoraproject DOT org + * Tim Bielawa - tbielawa AT gmail DOT com Send in a github pull request to get your name here. From 43f7dee2471d90e155bd8e0024855b6768e778c7 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 01:02:24 -0500 Subject: [PATCH 24/33] Added rough sketch of what I want for playbook support. Debug heavy at the moment as I figure out how logging/output might look. A couple of major TODO features also listed in the file. --- bin/ansible | 1 + examples/playbook.yml | 6 +- lib/ansible/playbook.py | 133 +++++++++++++++++++++++++++++++++------- 3 files changed, 116 insertions(+), 24 deletions(-) diff --git a/bin/ansible b/bin/ansible index 4254b0b7fb..92316fcd67 100755 --- a/bin/ansible +++ b/bin/ansible @@ -78,6 +78,7 @@ class Cli(object): ) else: return ansible.playbook.PlayBook( + playbook=options.playbook, module_path=options.module_path, remote_user=options.remote_user, remote_pass=sshpass, diff --git a/examples/playbook.yml b/examples/playbook.yml index 7b5ec9c5c4..8a4cf19aef 100644 --- a/examples/playbook.yml +++ b/examples/playbook.yml @@ -1,12 +1,12 @@ -- pattern: '*.prod.example.com' +- pattern: '*' tasks: - do: - - update apache (note: service module TBD) + - update apache - command - [/usr/bin/yum, update, apache] onchange: - do: - - restart apache (note: service module TBD) + - restart apache - command - [/sbin/service, apache, restart] - do: diff --git a/lib/ansible/playbook.py b/lib/ansible/playbook.py index fa6c27731e..e8bb15d9c4 100755 --- a/lib/ansible/playbook.py +++ b/lib/ansible/playbook.py @@ -30,7 +30,13 @@ import yaml class PlayBook(object): ''' runs an ansible playbook, given as a datastructure - or YAML filename + or YAML filename. a playbook is a deployment, config + management, or automation based set of commands to + run in series. + + multiple patterns do not execute simultaneously, + but tasks in each pattern do execute in parallel + according to the number of forks requested. ''' def __init__(self, @@ -44,31 +50,116 @@ class PlayBook(object): verbose=False): # runner is reused between calls - - self.runner = ansible.runner.Runner( - host_list=host_list, - module_path=module_path, - forks=forks, - timeout=timeout, - remote_user=remote_user, - remote_pass=remote_pass, - verbose=verbose - ) + + self.host_list = host_list + self.module_path = module_path + self.forks = forks + self.timeout = timeout + self.remote_user = remote_user + self.remote_pass = remote_pass + self.verbose = verbose if type(playbook) == str: playbook = yaml.load(file(playbook).read()) - + self.playbook = playbook + def run(self): - pass + ''' run against all patterns in the playbook ''' + + for pattern in self.playbook: + self._run_pattern(pattern) + return "complete" + + def _get_task_runner(self, + pattern=None, + host_list=None, + module_name=None, + module_args=None): + + print "GET TASK RUNNER FOR HL=%s" % host_list + + ''' + return a runner suitable for running this task, using + preferences from the constructor + ''' + + if host_list is None: + host_list = self.host_list + + return ansible.runner.Runner( + pattern=pattern, + module_name=module_name, + module_args=module_args, + host_list=host_list, + forks=self.forks, + remote_user=self.remote_user, + remote_pass=self.remote_pass, + module_path=self.module_path, + timeout=self.timeout + ) + + def _run_task(self, pattern, task, host_list=None): + ''' + run a single task in the playbook and + recursively run any subtasks. + ''' + + if host_list is None: + host_list = self.host_list + + print "TASK=%s" % task + instructions = task['do'] + (comment, module_name, module_args) = instructions + print "running task: (%s) on hosts matching (%s)" % (comment, pattern) + runner = self._get_task_runner( + pattern=pattern, + module_name=module_name, + module_args=module_args + ) + results = runner.run() + print "RESULTS=%s" % results + + dark = results.get("dark", []) + contacted = results.get("contacted", []) + + # TODO: filter based on values that indicate + # they have changed events to emulate Puppet + # 'notify' behavior, super easy -- just + # a list comprehension -- but we need complaint + # modules first + + ok_hosts = contacted.keys() + + for host, msg in dark.items(): + print "contacting %s failed -- %s" % (host, msg) + + subtasks = task.get('onchange', []) + if len(subtasks) > 0: + print "the following hosts have registered change events" + print ok_hosts + for subtask in subtasks: + self._run_task(pattern, subtask, ok_hosts) + + # TODO: if a host fails in task 1, add it to an excludes + # list such that no other tasks in the list ever execute + # unlike Puppet, do not allow partial failure of the tree + # and continuing as far as possible. Fail fast. + + + def _run_pattern(self, pg): + ''' + run a list of tasks for a given pattern, in order + ''' + + pattern = pg['pattern'] + tasks = pg['tasks'] + print "PATTERN=%s" % pattern + print "TASKS=%s" % tasks + for task in tasks: + print "*** RUNNING A TASK (%s)***" % task + self._run_task(pattern, task) + -# r = Runner( -# host_list = DEFAULT_HOST_LIST, -# module_name='ping', -# module_args='', -# pattern='*', -# forks=3 -# ) -# print r.run() From 4caf9d274bd5b172e42b7be4b6f90818b85755fa Mon Sep 17 00:00:00 2001 From: Tim Bielawa Date: Fri, 24 Feb 2012 20:07:30 -0500 Subject: [PATCH 25/33] Fix tbielawa email in AUTHORS file --- AUTHORS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS.md b/AUTHORS.md index eb5b5f301f..c920bd4782 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -4,7 +4,7 @@ Patches and Contributions * Michael DeHaan - michael.dehaan AT gmail DOT com * Jeremy Katz - katzj AT fedoraproject DOT org * Seth Vidal - skvidal AT fedoraproject DOT org - * Tim Bielawa - tbielawa AT gmail DOT com + * Tim Bielawa - tbielawa AT redhat DOT com Send in a github pull request to get your name here. From f0b021177289e4801111f1d27a2aa189eb041b15 Mon Sep 17 00:00:00 2001 From: Tim Bielawa Date: Fri, 24 Feb 2012 20:05:05 -0500 Subject: [PATCH 26/33] Because everything should have a man page --- .gitignore | 3 + Makefile | 32 +++++++++ docs/man/.gitignore | 1 + docs/man/man1/ansible.1 | 108 ++++++++++++++++++++++++++++++ docs/man/man1/ansible.1.asciidoc | 109 +++++++++++++++++++++++++++++++ setup.py | 4 +- 6 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 docs/man/.gitignore create mode 100644 docs/man/man1/ansible.1 create mode 100644 docs/man/man1/ansible.1.asciidoc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..d1a7b1604b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +*.py[co] +build diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..dca3a50308 --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +#!/usr/bin/make + +ASCII2MAN = a2x -D $(dir $@) -d manpage -f manpage $< +ASCII2HTMLMAN = a2x -D docs/html/man/ -d manpage -f xhtml +MANPAGES := docs/man/man1/ansible.1 +SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") + +docs: manuals + +manuals: $(MANPAGES) + +%.1: %.1.asciidoc + $(ASCII2MAN) + +%.5: %.5.asciidoc + $(ASCII2MAN) + +pep8: + @echo "#############################################" + @echo "# Running PEP8 Compliance Tests" + @echo "#############################################" + pep8 lib/ + +clean: + find . -type f -name "*.pyc" -delete + find . -type f -name "*.pyo" -delete + find . -type f -name "*~" -delete + find ./docs/ -type f -name "*.xml" -delete + find . -type f -name "#*" -delete + +.PHONEY: docs manual clean pep8 +vpath %.asciidoc docs/man/man1 diff --git a/docs/man/.gitignore b/docs/man/.gitignore new file mode 100644 index 0000000000..b81c7954b7 --- /dev/null +++ b/docs/man/.gitignore @@ -0,0 +1 @@ +*.xml \ No newline at end of file diff --git a/docs/man/man1/ansible.1 b/docs/man/man1/ansible.1 new file mode 100644 index 0000000000..7ff7233b0e --- /dev/null +++ b/docs/man/man1/ansible.1 @@ -0,0 +1,108 @@ +'\" t +.\" Title: ansible +.\" Author: [see the "AUTHOR" section] +.\" Generator: DocBook XSL Stylesheets v1.76.1 +.\" Date: 02/24/2012 +.\" Manual: System administration commands +.\" Source: Ansible 0.0.1 +.\" Language: English +.\" +.TH "ANSIBLE" "1" "02/24/2012" "Ansible 0\&.0\&.1" "System administration commands" +.\" ----------------------------------------------------------------- +.\" * Define some portability stuff +.\" ----------------------------------------------------------------- +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.\" http://bugs.debian.org/507673 +.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" ----------------------------------------------------------------- +.\" * set default formatting +.\" ----------------------------------------------------------------- +.\" disable hyphenation +.nh +.\" disable justification (adjust text to left margin only) +.ad l +.\" ----------------------------------------------------------------- +.\" * MAIN CONTENT STARTS HERE * +.\" ----------------------------------------------------------------- +.SH "NAME" +ansible \- run a command somewhere else +.SH "SYNOPSIS" +.sp +ansible [\-H hosts_path] [\-L library_path] [\-f forks] [\-n module_name] [\-a [args1 [args2 \&...]]] [\-p host_pattern] [\-u remote_user] +.SH "DESCRIPTION" +.sp +\fBAnsible\fR is an extra\-simple Python API for doing \*(Aqremote things\*(Aq over SSH\&. +.SH "OPTIONS" +.PP +\fB\-P\fR, \fB\-\-askpass\fR +.RS 4 +Ask the user to input the ssh password for connecting\&. +.RE +.PP +\fB\-H\fR, \fB\-\-host\-list\fR +.RS 4 +Path to hosts list\&. +.RE +.PP +\fB\-L\fR, \fB\-\-library\fR +.RS 4 +Path to module library\&. +.RE +.PP +\fB\-f\fR, \fB\-\-forks\fR +.RS 4 +Level of parallelism\&. Specify as an integer\&. +.RE +.PP +\fB\-n\fR, \fB\-\-name\fR +.RS 4 +Module name to execute\&. +.RE +.PP +\fB\-a\fR, \fB\-\-args\fR +.RS 4 +Arguments to module\&. +.RE +.PP +\fB\-p\fR, \fB\-\-pattern\fR +.RS 4 +Hostname pattern\&. Accepts shell\-like globs\&. +.RE +.PP +\fB\-r\fR, \fB\-\-run\-playbook\fR +.RS 4 +Playbook file to run\&. Replaces the +\fB\-n\fR +and +\fB\-a\fR +options\&. +.RE +.PP +\fB\-u\fR, \fB\-\-remote\-user\fR +.RS 4 +Remote user to connect as\&. Uses +\fIroot\fR +by default\&. +.RE +.SH "INVENTORY" +.sp +Ansible stores the hosts it can potentially operate on in an inventory file\&. The syntax is simple: one host per line\&. Organize your hosts into multiple groups by separating them into multiple inventory files\&. +.SH "FILES" +.sp +/etc/ansible/hosts \(em Default hosts file +.sp +/usr/share/ansible \(em Default module library +.SH "AUTHOR" +.sp +Ansible was originally written by Michael DeHaan\&. See the AUTHORS file for a complete list of contributors\&. +.SH "COPYRIGHT" +.sp +Copyright \(co 2012, Michael DeHaan +.sp +Ansible is released under the terms of the MIT license\&. +.SH "SEE ALSO" +.sp +Ansible home page: https://github\&.com/mpdehaan/ansible/ diff --git a/docs/man/man1/ansible.1.asciidoc b/docs/man/man1/ansible.1.asciidoc new file mode 100644 index 0000000000..ca5e1e9dff --- /dev/null +++ b/docs/man/man1/ansible.1.asciidoc @@ -0,0 +1,109 @@ +ansible(1) +========= +:doctype:manpage +:man source: Ansible +:man version: 0.0.1 +:man manual: System administration commands + +NAME +---- +ansible - run a command somewhere else + + +SYNOPSIS +-------- +ansible [-H hosts_path] [-L library_path] [-f forks] [-n module_name] + [-a [args1 [args2 ...]]] [-p host_pattern] [-u remote_user] + + +DESCRIPTION +----------- + +*Ansible* is an extra-simple Python API for doing \'remote things' over +SSH. + + +OPTIONS +------- + +*-P*, *--askpass*:: + +Ask the user to input the ssh password for connecting. + + +*-H*, *--host-list*:: + +Path to hosts list. + + +*-L*, *--library*:: + +Path to module library. + + +*-f*, *--forks*:: + +Level of parallelism. Specify as an integer. + + +*-n*, *--name*:: + +Module name to execute. + + +*-a*, *--args*:: + +Arguments to module. + + +*-p*, *--pattern*:: + +Hostname pattern. Accepts shell-like globs. + + +*-r*, *--run-playbook*:: + +Playbook file to run. Replaces the *-n* and *-a* options. + + +*-u*, *--remote-user*:: + +Remote user to connect as. Uses __root__ by default. + + +INVENTORY +--------- + +Ansible stores the hosts it can potentially operate on in an inventory +file. The syntax is simple: one host per line. Organize your hosts +into multiple groups by separating them into multiple inventory files. + + +FILES +----- + +/etc/ansible/hosts -- Default hosts file + +/usr/share/ansible -- Default module library + + +AUTHOR +------ + +Ansible was originally written by Michael DeHaan. See the AUTHORS file +for a complete list of contributors. + + +COPYRIGHT +--------- + +Copyright © 2012, Michael DeHaan + +Ansible is released under the terms of the MIT license. + + + +SEE ALSO +-------- + +Ansible home page: diff --git a/setup.py b/setup.py index 25f72ee5fd..0749f73a8f 100644 --- a/setup.py +++ b/setup.py @@ -19,10 +19,12 @@ setup(name='ansible', 'library/command', 'library/facter', 'library/copy', + ]), + ('man/man1', [ + 'docs/man/man1/ansible.1' ]) ], scripts=[ 'bin/ansible', ] ) - From c8fe53c4b94408f5ffab023139ee6c5929aab8ff Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 01:13:21 -0500 Subject: [PATCH 27/33] Update README.md to reflect that this is a tool, not just an API. Though the API is important. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 11309e3cf8..a54d0c8170 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ Ansible ======= -Ansible is a extra-simple Python API for doing 'remote things' over SSH. +Ansible is a extra-simple tool/API for doing 'parallel remote things' over SSH -- whether +executing commands, running declarative 'modules', or executing larger 'playbooks' that +can serve as a configuration management or deployment system. While [Func](http://fedorahosted.org/func), which I co-wrote, aspired to avoid using SSH and have it's own daemon infrastructure, From f17c4ca4b2b95dd250ca4916dc9415a85f4e1800 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 02:04:50 -0500 Subject: [PATCH 28/33] Added an 'ohai' module. Some weird JSON hackage to get it to work. --- README.md | 1 + library/ohai | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 library/ohai diff --git a/README.md b/README.md index a54d0c8170..e683493662 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ Modules include: * command -- runs commands, giving output, return codes, and run time info * ping - just returns if the system is up or not * facter - retrieves facts about the host OS + * ohai - similar to facter, but returns structured data * copy - add files to remote systems Playbooks diff --git a/library/ohai b/library/ohai new file mode 100644 index 0000000000..968c047236 --- /dev/null +++ b/library/ohai @@ -0,0 +1,12 @@ +#!/usr/bin/python + +# requires 'ohai' to be installed + +import json +import subprocess + +cmd = subprocess.Popen("/usr/bin/ohai", stdout=subprocess.PIPE, stderr=subprocess.PIPE) +out, err = cmd.communicate() + +# try to cleanup the JSON, for some reason facter --json doesn't need this hack +print json.dumps(json.loads(out)) From 9e931f323c716b8caef817f85653291f126de6a5 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 02:05:49 -0500 Subject: [PATCH 29/33] Update ohai module to run on older python clients. --- library/ohai | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/library/ohai b/library/ohai index 968c047236..7aefd06501 100644 --- a/library/ohai +++ b/library/ohai @@ -2,7 +2,11 @@ # requires 'ohai' to be installed -import json +try: + import json +except ImportError: + import simplejson as json + import subprocess cmd = subprocess.Popen("/usr/bin/ohai", stdout=subprocess.PIPE, stderr=subprocess.PIPE) From ee2fa721f19c93df1d17fad901fe5c06294714cb Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 02:14:22 -0500 Subject: [PATCH 30/33] Kinda have to pass the host list parameter --- lib/ansible/playbook.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ansible/playbook.py b/lib/ansible/playbook.py index e8bb15d9c4..4adc986a72 100755 --- a/lib/ansible/playbook.py +++ b/lib/ansible/playbook.py @@ -112,7 +112,8 @@ class PlayBook(object): (comment, module_name, module_args) = instructions print "running task: (%s) on hosts matching (%s)" % (comment, pattern) runner = self._get_task_runner( - pattern=pattern, + pattern=pattern, + host_list=host_list, module_name=module_name, module_args=module_args ) From e25bb2f8882d456a120fe642788e6565a4e11a58 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 02:36:38 -0500 Subject: [PATCH 31/33] Upgrade output for playbook runs --- lib/ansible/playbook.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/ansible/playbook.py b/lib/ansible/playbook.py index 4adc986a72..f2e92b3415 100755 --- a/lib/ansible/playbook.py +++ b/lib/ansible/playbook.py @@ -68,6 +68,10 @@ class PlayBook(object): for pattern in self.playbook: self._run_pattern(pattern) + + # TODO: return a summary of success & failure counts per node + # TODO: in bin/ancible, ensure return codes are appropriate + return "complete" def _get_task_runner(self, @@ -76,8 +80,6 @@ class PlayBook(object): module_name=None, module_args=None): - print "GET TASK RUNNER FOR HL=%s" % host_list - ''' return a runner suitable for running this task, using preferences from the constructor @@ -98,7 +100,7 @@ class PlayBook(object): timeout=self.timeout ) - def _run_task(self, pattern, task, host_list=None): + def _run_task(self, pattern, task, host_list=None, conditional=False): ''' run a single task in the playbook and recursively run any subtasks. @@ -107,10 +109,14 @@ class PlayBook(object): if host_list is None: host_list = self.host_list - print "TASK=%s" % task instructions = task['do'] (comment, module_name, module_args) = instructions - print "running task: (%s) on hosts matching (%s)" % (comment, pattern) + + namestr = "%s/%s" % (pattern, comment) + if conditional: + namestr = "subset/%s" % namestr + print "TASK [%s]" % namestr + runner = self._get_task_runner( pattern=pattern, host_list=host_list, @@ -118,9 +124,9 @@ class PlayBook(object): module_args=module_args ) results = runner.run() - print "RESULTS=%s" % results dark = results.get("dark", []) + contacted = results.get("contacted", []) # TODO: filter based on values that indicate @@ -132,14 +138,20 @@ class PlayBook(object): ok_hosts = contacted.keys() for host, msg in dark.items(): - print "contacting %s failed -- %s" % (host, msg) + print "DARK: [%s] => %s" % (host, msg) + + for host, results in contacted.items(): + if module_name == "command": + if results.get("rc", 0) != 0: + print "FAIL: [%s/%s] => %s" % (host, comment, results) + elif results.get("failed", 0) == 1: + print "FAIL: [%s/%s]" % (host, comment, results) + subtasks = task.get('onchange', []) if len(subtasks) > 0: - print "the following hosts have registered change events" - print ok_hosts for subtask in subtasks: - self._run_task(pattern, subtask, ok_hosts) + self._run_task(pattern, subtask, ok_hosts, conditional=True) # TODO: if a host fails in task 1, add it to an excludes # list such that no other tasks in the list ever execute @@ -154,10 +166,7 @@ class PlayBook(object): pattern = pg['pattern'] tasks = pg['tasks'] - print "PATTERN=%s" % pattern - print "TASKS=%s" % tasks for task in tasks: - print "*** RUNNING A TASK (%s)***" % task self._run_task(pattern, task) From 6bfc275204d1376dbd7bf188df3d33bec4f345ce Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 02:49:05 -0500 Subject: [PATCH 32/33] update TODO with ideas for playbook and latest plans --- TODO.md | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/TODO.md b/TODO.md index 07e7d79eae..89d1c04656 100644 --- a/TODO.md +++ b/TODO.md @@ -1,14 +1,38 @@ TODO list and plans =================== +Playbook TODO: + + * error codes and failure summaries + * create modules that return 'changed' attributes + * fail nodes on errors, i.e. remove from host list, rather than continuing to pound them + * further improve output + * more conditional capability + * very good logging + +General: + + * logging + * async options * modules for users, groups, and files, using puppet style ensure mechanics + * very simple option constructing/parsing for modules + * templating module (how might this work syntax wise?) with facter/ohai awareness + * probably could lay down a values.json file + * use copy capabilities to move files to tmp, run python templating + * maybe support templating engine of choice + * think about how to build idempotency guards around command module? + * think about how to feed extra JSON data onto system + +Bonus utilities: + * ansible-inventory - gathering fact/hw info, storing in git, adding RSS * ansible-slurp - recursively rsync file trees for each host - * very simple option constructing/parsing for modules - * Dead-simple declarative configuration management engine using - a runbook style recipe file, written in JSON or YAML * maybe it's own fact engine, not required, that also feeds from facter - * add/remove/list hosts from the command line - * list available modules from command line - * filter exclusion (run this only if fact is true/false) + +Not so interested really, but maybe: + + * list available modules from command line + * add/remove/list hosts from the command line + * filter exclusion (run this only if fact is true/false) + -- should be doable with playbooks (i.e. not neccessary) From 0de9f0b28ee777da3c5e5cbf33f5a1c9c445b3c2 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Fri, 24 Feb 2012 03:04:46 -0500 Subject: [PATCH 33/33] Added idea about how to do async + timeout in modules --- TODO.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TODO.md b/TODO.md index 89d1c04656..8fd5ef73ac 100644 --- a/TODO.md +++ b/TODO.md @@ -10,6 +10,12 @@ Playbook TODO: * more conditional capability * very good logging +Command module: + * allow additional key/value options to be passed to any module (via ENV vars?) + * allow this to be expressed in playbook as a 4th option after the array options list + * use this to pass timeout and async params to the command module + default timeouts will be infinite, async False + General: * logging