The very first start of 'fireball mode', which uses ansible in SSH mode to deploy a ephemeral zeromq daemon (entirely optional) that is will die after

it is no longer used.  No key signing will be required.  At this point, development use only -- NOT complete.

lib/ansible/constants.py

@@ -85,3 +85,4 @@ DEFAULT_SUDO_PASS         = None
 DEFAULT_SUBSET            = None
 
 ANSIBLE_SSH_ARGS          = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', None)
+ZEROMQ_PORT               = int(get_config(p, 'fireball', 'zeromq_port', 'ANSIBLE_ZEROMQ_PORT', 5099))

lib/ansible/module_common.py

@@ -206,6 +206,7 @@ class AnsibleModule(object):
 
     def _log_invocation(self):
         ''' log that ansible ran the module '''
+        # TODO: generalize a seperate log function and make log_invocation use it
         # Sanitize possible password argument when logging.
         log_args = dict()
         passwd_keys = ['password', 'login_password']

lib/ansible/runner/__init__.py

@@ -26,6 +26,7 @@ import tempfile
 import time
 import collections
 import socket
+import base64
 
 import ansible.constants as C
 import ansible.inventory
@@ -45,7 +46,8 @@ except ImportError:
 
 dirname = os.path.dirname(__file__)
 action_plugin_list = utils.import_plugins(os.path.join(dirname, 'action_plugins'))
-
+    
+        
 ################################################
 
 def _executor_hook(job_queue, result_queue):
@@ -153,11 +155,11 @@ class Runner(object):
             # ability to turn off temp file deletion for debug purposes
             return
 
-        if type(files) == str:
+        if type(files) in [ str, unicode ]:
             files = [ files ]
         for filename in files:
             if filename.find('/tmp/') == -1:
-                raise Exception("not going to happen")
+                raise Exception("safeguard deletion, removal of %s is not going to happen" % filename)
             self._low_level_exec_command(conn, "rm -rf %s" % filename, None)
 
     # *****************************************************
@@ -188,6 +190,10 @@ class Runner(object):
 
         ''' runs a module that has already been transferred '''
 
+        # hack to support fireball mode
+        if module_name == 'fireball':
+            args = "%s password=%s port=%s" % (args, base64.b64encode(str(utils.key_for_hostname(conn.host))), C.ZEROMQ_PORT)
+
         (remote_module_path, is_new_style) = self._copy_module(conn, tmp, module_name, args, inject)
         cmd = "chmod u+x %s" % remote_module_path
         if self.sudo and self.sudo_user != 'root':
@@ -404,12 +410,12 @@ class Runner(object):
         sudo_user = self.sudo_user
         stdin, stdout, stderr = conn.exec_command(cmd, tmp, sudo_user, sudoable=sudoable)
 
-        if type(stdout) != str:
+        if type(stdout) not in [ str, unicode ]:
             out = "\n".join(stdout.readlines())
         else:
             out = stdout
 
-        if type(stderr) != str:
+        if type(stderr) not in [ str, unicode ]:
             err = "\n".join(stderr.readlines())
         else:
             err = stderr
@@ -452,7 +458,9 @@ class Runner(object):
         cmd += ' && echo %s' % basetmp
 
         result = self._low_level_exec_command(conn, cmd, None, sudoable=False)
-        return utils.last_non_blank_line(result).strip() + '/'
+        rc = utils.last_non_blank_line(result).strip() + '/'
+        return rc
+
 
     # *****************************************************
 
@@ -499,9 +507,10 @@ class Runner(object):
     def _parallel_exec(self, hosts):
         ''' handles mulitprocessing when more than 1 fork is required '''
 
-        job_queue = multiprocessing.Manager().Queue()
+        manager = multiprocessing.Manager()
+        job_queue = manager.Queue()
         [job_queue.put(i) for i in hosts]
-        result_queue = multiprocessing.Manager().Queue()
+        result_queue = manager.Queue()
 
         workers = []
         for i in range(self.forks):

lib/ansible/runner/connection_plugins/fireball.py

@@ -0,0 +1,129 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+from ansible.callbacks import vvv
+from ansible import utils
+from ansible import errors
+from ansible import constants
+
+HAVE_ZMQ=False
+
+try:
+    import zmq
+    HAVE_ZMQ=True
+except ImportError:
+    pass
+
+class Connection(object):
+    ''' SSH based connections with Paramiko '''
+
+    def __init__(self, runner, host, port=None):
+
+        self.runner = runner
+
+        # attempt to work around shared-memory funness
+        if getattr(self.runner, 'aes_keys', None):
+            utils.AES_KEYS = self.runner.aes_keys
+
+        self.host = host
+        self.key = utils.key_for_hostname(host)
+        self.socket = None
+        # port passed in is the SSH port, which we ignore
+        self.port = constants.ZEROMQ_PORT
+  
+    def connect(self):
+        ''' activates the connection object '''
+
+        if not HAVE_ZMQ:
+            raise errors.AnsibleError("zmq is not installed")
+        
+        # this is rough/temporary and will likely be optimized later ...
+        context = zmq.Context()
+        socket = context.socket(zmq.REQ)
+        addr = "tcp://%s:%s" % (self.host, self.port)
+        socket.connect(addr)
+        self.socket = socket    
+
+        return self
+
+    def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False):
+        ''' run a command on the remote host '''
+
+        vvv("EXEC COMMAND %s" % cmd)
+
+        if self.runner.sudo and sudoable:
+            raise errors.AnsibleError("fireball does not use sudo, but runs as whoever it was initiated as.  (That itself is where to use sudo).")
+
+        data = dict(
+            mode='command',
+            cmd=cmd,
+            tmp_path=tmp_path,
+        )
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        self.socket.send(data)
+        
+        response = self.socket.recv()
+        response = utils.decrypt(self.key, response)
+        response = utils.parse_json(response)
+
+        return ('', response.get('stdout',''), response.get('stderr',''))
+
+    def put_file(self, in_path, out_path):
+
+        ''' transfer a file from local to remote '''
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
+
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        data = file(in_path).read()
+        
+        data = dict(mode='put', data=data, out_path=out_path)
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        self.socket.send(data)
+
+        response = self.socket.recv()
+        response = utils.decrypt(self.key, response)
+        response = utils.parse_json(response)
+
+        # no meaningful response needed for this
+
+    def fetch_file(self, in_path, out_path):
+        ''' save a remote file to the specified path '''
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
+
+        data = dict(mode='fetch', file=in_path)
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        self.socket.send(data)
+
+        response = self.socket.recv()
+        response = utils.decrypt(self.key, response)
+        response = utils.parse_json(response)
+        response = response['data']
+
+        fh = open(out_path, "w")
+        fh.write(response)
+        fh.close()
+
+    def close(self):
+        ''' terminate the connection '''
+        # no need for this
+

lib/ansible/utils.py

@@ -35,6 +35,7 @@ import subprocess
 import stat
 import termios
 import tty
+from multiprocessing import Manager
 
 VERBOSITY=0
 
@@ -48,15 +49,54 @@ try:
 except ImportError:
     from md5 import md5 as _md5
 
-# vars_prompt_encrypt
 PASSLIB_AVAILABLE = False
-
 try:
     import passlib.hash
     PASSLIB_AVAILABLE = True
 except:
     pass
 
+KEYCZAR_AVAILABLE=False
+try:
+    from keyczar.keys import AesKey
+    KEYCZAR_AVAILABLE=True
+except ImportError:
+    pass
+
+###############################################################
+# abtractions around keyczar
+
+def key_for_hostname(hostname):
+    # fireball mode is an implementation of ansible firing up zeromq via SSH
+    # to use no persistent daemons or key management
+
+    key_path = os.path.expanduser("~/.fireball.keys")
+    if not os.path.exists(key_path):
+        os.makedirs(key_path)
+    key_path = os.path.expanduser("~/.fireball.keys/%s" % hostname)
+
+    # use new AES keys every 2 hours, which means fireball must not allow running for longer either
+    if not os.path.exists(key_path) or (time.time() - os.path.getmtime(key_path) > 60*60*2):
+        key = AesKey.Generate()
+        fh = open(key_path, "w")
+        fh.write(str(key))
+        fh.close()
+        return key
+    else:
+        fh = open(key_path)
+        key = AesKey.Read(fh.read())  
+        fh.close()
+        return key
+
+def encrypt(key, msg):
+    return key.Encrypt(msg)
+
+def decrypt(key, msg):
+    try:
+        return key.Decrypt(msg)
+    except keyczar.errors.InvalidSignatureError:
+        raise errors.AnsibleError("decryption failed")
+
 ###############################################################
 # UTILITY FUNCTIONS FOR COMMAND LINE TOOLS
 ###############################################################

library/fireball

@@ -0,0 +1,216 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import shutil
+import time
+import base64
+import syslog
+import signal
+import subprocess
+
+syslog.openlog('ansible-%s' % os.path.basename(__file__))
+PIDFILE = os.path.expanduser("~/.fireball.pid")
+
+def log(msg):
+   syslog.syslog(syslog.LOG_NOTICE, msg)
+
+if os.path.exists(PIDFILE):
+    try:
+        data = int(open(PIDFILE).read())
+        try:
+            os.kill(data, signal.SIGKILL)
+        except OSError:
+            pass
+    except ValueError:
+        pass
+    os.unlink(PIDFILE)
+
+HAS_ZMQ = False
+try:
+   import zmq
+   HAS_ZMQ = True
+except ImportError:
+   pass
+
+HAS_KEYCZAR = False
+try:
+   from keyczar.keys import AesKey
+   HAS_KEYCZAR = True
+except ImportError:
+   pass
+
+# NOTE: this shares a fair amount of code in common with async_wrapper, if async_wrapper were a new module we could move
+# this into utils.module_common and probably should anyway
+
+def daemonize_self(module, password, port, minutes):
+    # daemonizing code: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012
+    try:
+        pid = os.fork()
+        if pid > 0:
+            log("exiting pid %s" % pid)
+            # exit first parent
+            module.exit_json(msg="daemonzed fireball on port %s for %s minutes" % (port, minutes))
+    except OSError, e:
+        log("fork #1 failed: %d (%s)" % (e.errno, e.strerror))
+        sys.exit(1)
+
+    # decouple from parent environment
+    os.chdir("/")
+    os.setsid()
+    os.umask(022)
+
+    # do second fork
+    try:
+        pid = os.fork()
+        if pid > 0:
+            log("daemon pid %s, writing %s" % (pid, PIDFILE))
+            pid_file = open(PIDFILE, "w")
+            pid_file.write("%s" % pid)
+            pid_file.close()
+            log("pidfile written")
+            sys.exit(0)
+    except OSError, e:
+        log("fork #2 failed: %d (%s)" % (e.errno, e.strerror))
+        sys.exit(1)
+
+    dev_null = file('/dev/null','rw')
+    os.dup2(dev_null.fileno(), sys.stdin.fileno())
+    os.dup2(dev_null.fileno(), sys.stdout.fileno())
+    os.dup2(dev_null.fileno(), sys.stderr.fileno())
+    log("daemonizing successful (%s,%s)" % (password, port))
+
+def command(data):
+    if 'cmd' not in data:
+        return dict(failed=True, msg='internal error: cmd is required')
+    if 'tmp_path' not in data:
+        return dict(failed=True, msg='internal error: tmp_path is required')
+
+    log("executing: %s" % data['cmd'])
+    p = subprocess.Popen(data['cmd'], shell=True, stdout=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    if stdout is None:
+        stdout = ''
+    if stderr is None:
+        stderr = ''
+    log("got stdout: %s" % stdout)
+
+    return dict(stdout=stdout, stderr=stderr)
+  
+def fetch(data):
+    if 'data' not in data:
+       return dict(failed=True, msg='internal error: data is required')
+    if 'in_path' not in data:
+       return dict(failed=True, msg='internal error: out_path is required')
+
+    fh = open(data['in_path'])
+    data = fh.read()
+    return dict(data=data)
+
+def put(data):
+
+    if 'data' not in data:
+        return dict(failed=True, msg='internal error: data is required')
+    if 'out_path' not in data:
+        return dict(failed=True, msg='internal error: out_path is required')
+
+    fh = open(data['out_path'], 'w')
+    fh.write(data['data'])
+    fh.close()
+
+    return dict()
+
+def serve(module, password, port, minutes):
+
+    log("serving")
+    context = zmq.Context()
+    socket = context.socket(zmq.REP)
+    addr = "tcp://*:%s" % port
+    log("zmq serving on %s" % addr)
+    socket.bind(addr)
+
+    # password isn't so much a password but a serialized AesKey object that we xferred over SSH
+    # password as a variable in ansible is never logged though, so it serves well
+
+    key = AesKey.Read(password)
+    log("DEBUG KEY=%s" % key) # REALLY NEED TO REMOVE THIS, DEBUG/DEV ONLY!
+ 
+    while True:
+
+        log("DEBUG: waiting")
+        data = socket.recv()
+        data = key.Decrypt(data)
+        data = json.loads(data)
+        log("DEBUG: got data=%s" % data)
+
+        mode = data['mode']
+        response = {}
+
+        if mode == 'command':
+            response = command(data)
+        elif mode == 'put':
+            response = put(data)
+        elif mode == 'fetch':
+            response = fetch(data)
+
+        # FIXME: send back a useful response here
+        data2 = json.dumps(response)
+        log("DEBUG: returning data=%s" % data2)
+        data2 = key.Encrypt(data2)
+        socket.send(data2)
+
+def daemonize(module, password, port, minutes):
+
+    # FIXME: actually support the minutes killswitch here
+    # FIXME: /actually/ daemonize here
+    try:
+        daemonize_self(module, password, port, minutes)
+        serve(module, password, port, minutes)
+    except Exception, e:
+        log("exception caught, exiting fireball mode: %s" % e)
+        sys.exit(0)
+
+def main():
+
+    module = AnsibleModule(
+        argument_spec = dict(
+            port=dict(required=False, default=5099),
+            password=dict(required=True),
+            minutes=dict(required=False, default=30),
+        )
+    )
+
+    password  = base64.b64decode(module.params['password'])
+    log("DEBUG pass=%s" % password)
+    port      = module.params['port']
+    minutes   = module.params['minutes']
+
+    if not HAS_ZMQ:
+        module.fail_json(msg="zmq is not installed")
+    if not HAS_KEYCZAR:
+        module.fail_json(msg="keyczar is not installed")    
+
+    daemonize(module, password, port, minutes)
+   
+
+# this is magic, see lib/ansible/module_common.py
+#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
+main()