1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00

Merge branch 'optimise_utils_clean_data' of https://github.com/leth/ansible into leth-optimise_utils_clean_data

This commit is contained in:
James Cammarata 2014-08-11 10:21:02 -05:00
commit 25e9d1197e
2 changed files with 42 additions and 51 deletions

View file

@ -46,6 +46,7 @@ import getpass
import sys
import json
import subprocess
import contextlib
from vault import VaultLib
@ -56,6 +57,8 @@ MAX_FILE_SIZE_FOR_DIFF=1*1024*1024
# caching the compilation of the regex used
# to check for lookup calls within data
LOOKUP_REGEX = re.compile(r'lookup\s*\(')
PRINT_CODE_REGEX = re.compile(r'(?:{[{%]|[%}]})')
CODE_REGEX = re.compile(r'(?:{%|%})')
try:
import json
@ -355,64 +358,48 @@ def _clean_data(orig_data, from_remote=False, from_inventory=False):
if not isinstance(orig_data, basestring):
return orig_data
data = StringIO.StringIO("")
# when the data is marked as having come from a remote, we always
# replace any print blocks (ie. {{var}}), however when marked as coming
# from inventory we only replace print blocks that contain a call to
# a lookup plugin (ie. {{lookup('foo','bar'))}})
replace_prints = from_remote or (from_inventory and '{{' in orig_data and LOOKUP_REGEX.search(orig_data) is not None)
regex = PRINT_CODE_REGEX if replace_prints else CODE_REGEX
with contextlib.closing(StringIO.StringIO(orig_data)) as data:
# these variables keep track of opening block locations, as we only
# want to replace matched pairs of print/block tags
print_openings = []
block_openings = []
for mo in regex.finditer(orig_data):
token = mo.group(0)
token_start = mo.start(0)
for idx,c in enumerate(orig_data):
# if the current character is an opening brace, check to
# see if this is a jinja2 token. Otherwise, if the current
# character is a closing brace, we backup one character to
# see if we have a closing.
if c == '{' and idx < len(orig_data) - 1:
token = orig_data[idx:idx+2]
# if so, and we want to replace this block, push
# this token's location onto the appropriate array
if token == '{{' and replace_prints:
print_openings.append(idx)
elif token == '{%':
block_openings.append(idx)
# finally we write the data to the buffer and write
data.seek(0, os.SEEK_END)
data.write(c)
elif c == '}' and idx > 0:
token = orig_data[idx-1:idx+1]
prev_idx = -1
if token == '%}' and len(block_openings) > 0:
if token[0] == '{':
if token == '{%':
block_openings.append(token_start)
elif token == '{{':
print_openings.append(token_start)
elif token[1] == '}':
prev_idx = None
if token == '%}' and block_openings:
prev_idx = block_openings.pop()
elif token == '}}' and len(print_openings) > 0:
elif token == '}}' and print_openings:
prev_idx = print_openings.pop()
# if we have a closing token, and we have previously found
# the opening to the same kind of block represented by this
# token, replace both occurrences, otherwise we just write
# the current character to the buffer
if prev_idx != -1:
if prev_idx is not None:
# replace the opening
data.seek(prev_idx, os.SEEK_SET)
data.write('{#')
# replace the closing
data.seek(-1, os.SEEK_END)
data.seek(token_start, os.SEEK_SET)
data.write('#}')
else:
data.seek(0, os.SEEK_END)
data.write(c)
else:
# not a jinja2 token, so we just write the current char
# to the output buffer
data.seek(0, os.SEEK_END)
data.write(c)
return_data = data.getvalue()
data.close()
return return_data
assert False, 'Unhandled regex match'
return data.getvalue()
def _clean_data_struct(orig_data, from_remote=False, from_inventory=False):
'''

View file

@ -729,6 +729,10 @@ class TestUtils(unittest.TestCase):
ansible.utils._clean_data('this string has a {{variable}}', from_remote=True),
'this string has a {#variable#}'
)
self.assertEqual(
ansible.utils._clean_data('this string {{has}} two {{variables}} in it', from_remote=True),
'this string {#has#} two {#variables#} in it'
)
self.assertEqual(
ansible.utils._clean_data('this string has a {{variable with a\nnewline}}', from_remote=True),
'this string has a {#variable with a\nnewline#}'