Using custom splitting function for module param counting

lib/ansible/module_utils/basic.py

@@ -218,6 +218,106 @@ def load_platform_subclass(cls, *args, **kwargs):
 
     return super(cls, subclass).__new__(subclass)
 
+def split_args(args):
+    '''
+    Splits args on whitespace, but intelligently reassembles
+    those that may have been split over a jinja2 block or quotes.
+    When used in a module, we won't ever have to be concerned about
+    jinja2 blocks, however this function is/will be used in the
+    core portions as well before the args are templated.
+    '''
+
+    # the list of params parsed out of the arg string
+    params = []
+    # here we encode the args, so we have a uniform charset to
+    # work with, and split on white space
+    args = args.encode('utf-8')
+    items = args.split()
+
+    # iterate over the items, and reassemble any that may have been
+    # split on a space inside a jinja2 block. These variables are used
+    # to keep track of the state of the parsing, since blocks and quotes
+    # may be nested within each other.
+    inside_quotes = False
+    quote_char = None
+    split_print_depth = 0
+    split_block_depth = 0
+    split_comment_depth = 0
+    # now we loop over each split item, coalescing items if the white space
+    # split occurred within quotes or a jinja2 block of some kind
+    for item in items:
+        item = item.strip()
+        # store the previous quoting state for checking later
+        was_inside_quotes = inside_quotes
+        # determine the current quoting state
+        for i in range(0, len(item)):
+            c = item[i]
+            bc = None
+            if i > 0:
+                bc = item[i-1]
+            if c in ('"', "'"):
+                if inside_quotes:
+                    if c == quote_char and bc != '\\':
+                        inside_quotes = False
+                        quote_char = None
+                else:
+                    inside_quotes = True
+                    quote_char = c
+        # multiple conditions may append a token to the list of params,
+        # so we keep track with this flag to make sure it only happens once
+        appended = False
+        # if we're inside quotes now, but weren't before, append the item
+        # to the end of the list, since we'll tack on more to it later
+        if inside_quotes and not was_inside_quotes:
+            params.append(item)
+            appended = True
+        # otherwise, if we're inside any jinja2 block, inside quotes, or we were
+        # inside quotes (but aren't now) concat this item to the last param
+        elif ((split_print_depth + split_block_depth + split_comment_depth) > 0 or inside_quotes or was_inside_quotes):
+            params[-1] = "%s %s" % (params[-1], item)
+            appended = True
+        # these variables are used to determine the current depth of each jinja2
+        # block type, by counting the number of openings and closing tags
+        num_print_open    = item.count('{{')
+        num_print_close   = item.count('}}')
+        num_block_open    = item.count('{%')
+        num_block_close   = item.count('%}')
+        num_comment_open  = item.count('{#')
+        num_comment_close = item.count('#}')
+        # if the number is not the same, the depth has changed, so we calculate that here
+        # and may append the current item to the params (if we haven't previously done so)
+        if num_print_open != num_print_close:
+            split_print_depth += (num_print_open - num_print_close)
+            if not appended:
+                params.append(item)
+                appended = True
+            if split_print_depth < 0:
+                split_print_depth = 0
+        if num_block_open != num_block_close:
+            split_block_depth += (num_block_open - num_block_close)
+            if not appended:
+                params.append(item)
+                appended = True
+            if split_block_depth < 0:
+                split_block_depth = 0
+        if num_comment_open != num_comment_close:
+            split_comment_depth += (num_comment_open - num_comment_close)
+            if not appended:
+                params.append(item)
+                appended = True
+            if split_comment_depth < 0:
+                split_comment_depth = 0
+        # finally, if we're at zero depth for all blocks and not inside quotes, and have not
+        # yet appended anything to the list of params, we do so now
+        if (split_print_depth + split_block_depth + split_comment_depth) == 0 and not inside_quotes and not appended:
+            params.append(item)
+    # If we're done and things are not at zero depth or we're still inside quotes,
+    # raise an error to indicate that the args were unbalanced
+    if (split_print_depth + split_block_depth + split_comment_depth) != 0 or inside_quotes:
+        raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes")
+    # finally, we decode each param back to the unicode it was in the arg string
+    params = [x.decode('utf-8') for x in params]
+    return params
 
 class AnsibleModule(object):
 

lib/ansible/runner/__init__.py

@@ -47,6 +47,7 @@ import connection
 from return_data import ReturnData
 from ansible.callbacks import DefaultRunnerCallbacks, vv
 from ansible.module_common import ModuleReplacer
+from ansible.module_utils.basic import split_args
 
 module_replacer = ModuleReplacer(strip_comments=False)
 
@@ -397,14 +398,10 @@ class Runner(object):
         '''
         options = {}
         if args is not None:
-            args = args.encode('utf-8')
             try:
-                lexer = shlex.shlex(args)
+                vargs = split_args(args)
-                lexer.whitespace = '\t '
+            except Exception, e:
-                lexer.whitespace_split = True
+                if "unbalanced jinja2 block or quotes" in str(e):
-                vargs = [x.decode('utf-8') for x in lexer]
-            except ValueError, ve:
-                if 'no closing quotation' in str(ve).lower():
                     raise errors.AnsibleError("error parsing argument string '%s', try quoting the entire line." % args)
                 else:
                     raise

lib/ansible/utils/__init__.py

@@ -30,6 +30,7 @@ from ansible.utils import template
 from ansible.utils.display_functions import *
 from ansible.utils.plugins import *
 from ansible.callbacks import display
+from ansible.module_utils.basic import split_args
 import ansible.constants as C
 import ast
 import time

test/integration/roles/test_lookups/tasks/main.yml

@@ -92,7 +92,7 @@
 
 # https://github.com/ansible/ansible/issues/6550
 - name: confirm pipe lookup works with multiple positional args
-  debug: msg="{{ lookup('pipe', 'ls /tmp /') }}"         
+  debug: msg="{{ lookup('pipe', 'ls -l /tmp') }}"