When playbooks fail, attempt to create an inventory file in the inventory directory that allows rerunning

of the playbook against only the hosts that failed.

CHANGELOG.md

@@ -13,6 +13,7 @@ Core Features:
 * can set ansible_private_key_file as an inventory variable (similar to ansible_ssh_host, etc)
 * 'when' statement can be affixed to task includes to auto-affix the conditional to each task therein
 * cosmetic: "*****" banners in ansible-playbook output are now constant width
+* attempt to create an inventory file to rerun against failed hosts only, without retrying successful ones
 
 Modules added
 

bin/ansible-playbook

@@ -174,6 +174,7 @@ def main(args):
             print 'Playbook Syntax is fine'
             return 0
             
+        failed_hosts = []
 
         try:
 
@@ -182,6 +183,17 @@ def main(args):
             hosts = sorted(pb.stats.processed.keys())
             print callbacks.banner("PLAY RECAP")
             playbook_cb.on_stats(pb.stats)
+
+            for h in hosts:
+                t = pb.stats.summarize(h)
+                if t['unreachable'] > 0 or t['failures'] > 0:
+                    failed_hosts.append(h)
+
+            if len(failed_hosts) > 0:
+                filename = pb.generate_retry_inventory(failed_hosts)
+                if filename:
+                    print "           to rerun against failed hosts only, use -i %s\n" % filename
+
             for h in hosts:
                 t = pb.stats.summarize(h)
                 print "%s : %s %s %s %s" % (
@@ -191,16 +203,15 @@ def main(args):
                     colorize('unreachable', t['unreachable'], 'red'),
                     colorize('failed', t['failures'], 'red'))
  
-            print "\n"
-            for h in hosts:
-                stats = pb.stats.summarize(h)
-                if stats['failures'] != 0 or stats['unreachable'] != 0:
+            print ""
+            if len(failed_hosts) > 0:
                 return 2
 
         except errors.AnsibleError, e:
             print >>sys.stderr, "ERROR: %s" % e
             return 1
 
+
     return 0
 
 

lib/ansible/inventory/dir.py

@@ -39,6 +39,10 @@ class InventoryDirectory(object):
         for i in self.names:
             if i.endswith("~") or i.endswith(".orig") or i.endswith(".bak"):
                 continue
+            if i.endswith(".retry"):
+                # this file is generated on a failed playbook and should only be
+                # used when run specifically
+                continue
             # These are things inside of an inventory basedir
             if i in ("host_vars", "group_vars", "vars_plugins"):
                 continue

lib/ansible/playbook/__init__.py

@@ -25,6 +25,8 @@ import os
 import shlex
 import collections
 from play import Play
+import StringIO
+import pipes
 
 SETUP_CACHE = collections.defaultdict(dict)
 
@@ -129,6 +131,7 @@ class PlayBook(object):
         vars = {}
         if self.inventory.basedir() is not None:
             vars['inventory_dir'] = self.inventory.basedir()
+        self.filename = playbook
         (self.playbook, self.play_basedirs) = self._load_playbook_from_file(playbook, vars)
 
     # *****************************************************
@@ -415,6 +418,90 @@ class PlayBook(object):
 
     # *****************************************************
 
+
+    def generate_retry_inventory(self, replay_hosts):
+        '''
+        called by /usr/bin/ansible when a playbook run fails. It generates a inventory 
+        that allows re-running on ONLY the failed hosts.  This may duplicate some
+        variable information in group_vars/host_vars but that is ok, and expected.
+        ''' 
+
+        # TODO: move this into an inventory.serialize() method
+
+        buf = StringIO.StringIO()
+
+        buf.write("# dynamically generated inventory file\n")
+        buf.write("# retries previously failed hosts only\n")
+        buf.write("\n")
+
+        inventory = self.inventory
+        basedir = inventory.basedir()
+        filename = ".%s.retry" % os.path.basename(self.filename)
+        filename = os.path.join(basedir, filename)
+
+        def _simple_kv_vars(host_vars):
+            buf = ""
+            for (k, v) in host_vars.items():
+                if type(v) not in [ list, dict ]:
+                   if isinstance(v,basestring):
+                       buf = buf + " %s=%s" % (k, pipes.quote(v))
+                   else:
+                       buf = buf + " %s=%s" % (k, v)
+            return buf
+ 
+        # for all group names
+        for gname in inventory.groups_list():
+
+            # write the group name
+            group = inventory.get_group(gname)
+            group_vars = inventory.get_group_variables(gname)
+
+            # but only contain hosts that we want to replay
+            hostz = [ host.name for host in group.hosts ]
+            hostz = [ hname for hname in hostz if hname in replay_hosts ] 
+            if len(hostz):
+                buf.write("[%s]\n" % group.name)
+                for hostname in hostz:
+                    host = inventory.get_host(hostname)
+                    host_vars = host.vars
+                    hostname_vars = _simple_kv_vars(host_vars)
+                    buf.write("%s %s\n" % (hostname, hostname_vars))
+                buf.write("\n")
+
+            # write out any child groups if present
+            if len(group.child_groups) and group.name not in [ 'all', 'ungrouped' ]:
+                buf.write("\n")
+                buf.write("[%s:children]\n" % gname)
+                for child_group in group.child_groups:
+                    buf.write("%s\n" % child_group.name)
+                buf.write("\n")
+
+            # we do NOT write out group variables because they will have already
+            # been blended with the host
+
+            if len(group_vars.keys()) > 0 and group.name not in [ 'all', 'ungrouped' ]:
+                buf.write("[%s:vars]\n" % gname) 
+                for (k,v) in group_vars.items():
+                    if type(v) not in [list,dict]:
+                        if isinstance(type(k), basestring):
+                            buf.write("%s='%s'\n" % (k,v))  
+                        else:  
+                            buf.write("%s=%s\n" % (k,v))
+                buf.write("\n")
+
+        # if file isn't writeable, don't do anything.
+        # TODO: allow a environment variable to pick a different destination for this file
+
+        try:
+            fd = open(filename, 'w')
+            fd.write(buf.getvalue())
+            fd.close()
+            return filename
+        except Exception, e:
+            return None
+
+    # *****************************************************
+
     def _run_play(self, play):
         ''' run a list of tasks for a given pattern, in order '''