ansible/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00
Code Releases Activity

Cleanup basic.py code now that six is available (#17158)

Browse source 
* Cleanup basic.py code now that six is available

We had some hacks in basic.py to allow us python2 and python3
compatibility.  Those can now be offloaded to the six library that we're
bundling.

* Cleanup basic.py code now that six is available

We had some hacks in basic.py to allow us python2 and python3
compatibility.  Those can now be offloaded to the six library that we're
bundling.
This commit is contained in:
Toshio Kuratomi 2016-08-20 08:08:59 -07:00 committed by GitHub
parent 959a5e5fd1
commit 171a094805
1 changed files with 125 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

256
lib/ansible/module_utils/basic.py
View file

@ -63,55 +63,17 @@ except ImportError:
HAS_SYSLOG=False HAS_SYSLOG=False
try: try:
# Python 2 from systemd import journal
from itertools import imap has_journal = True
except ImportError: except ImportError:
# Python 3 has_journal = False
imap = map
HAVE_SELINUX=False
try: try:
# Python 2 import selinux
basestring HAVE_SELINUX=True
except NameError: except ImportError:
# Python 3 pass
basestring = str
try:
# Python 2
unicode
except NameError:
# Python 3
unicode = str
try:
# Python 2.6+
bytes
except NameError:
# Python 2.4
bytes = str
try:
dict.iteritems
except AttributeError:
# Python 3
def iteritems(d):
return d.items()
else:
# Python 2
def iteritems(d):
return d.iteritems()
try:
reduce
except NameError:
# Python 3
from functools import reduce
try:
NUMBERTYPES = (int, long, float)
except NameError:
# Python 3
NUMBERTYPES = (int, float)
# Python2 & 3 way to get NoneType # Python2 & 3 way to get NoneType
NoneType = type(None) NoneType = type(None)
@ -148,21 +110,6 @@ except ImportError:
print('\n{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}') print('\n{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}')
sys.exit(1) sys.exit(1)
from ansible.module_utils.six import PY2, PY3, b, binary_type, text_type, string_types
HAVE_SELINUX=False
try:
import selinux
HAVE_SELINUX=True
except ImportError:
pass
try:
from systemd import journal
has_journal = True
except ImportError:
has_journal = False
AVAILABLE_HASH_ALGORITHMS = dict() AVAILABLE_HASH_ALGORITHMS = dict()
try: try:
import hashlib import hashlib
@ -187,6 +134,7 @@ except ImportError:
pass pass
try: try:
# Python 2.6+
from ast import literal_eval from ast import literal_eval
except ImportError: except ImportError:
# a replacement for literal_eval that works with python 2.4. from: # a replacement for literal_eval that works with python 2.4. from:
@ -203,6 +151,8 @@ except ImportError:
and None. and None.
""" """
_safe_names = {'None': None, 'True': True, 'False': False} _safe_names = {'None': None, 'True': True, 'False': False}
# Okay to use basestring and long here because this is only for
# python 2.4 and 2.5
if isinstance(node_or_string, basestring): if isinstance(node_or_string, basestring):
node_or_string = parse(node_or_string, mode='eval') node_or_string = parse(node_or_string, mode='eval')
if isinstance(node_or_string, ast.Expression): if isinstance(node_or_string, ast.Expression):
@ -227,9 +177,43 @@ except ImportError:
_literal_eval = literal_eval _literal_eval = literal_eval
# Backwards compat. There were present in basic.py before from ansible.module_utils.six import (PY2, PY3, b, binary_type, integer_types,
iteritems, text_type, string_types)
from ansible.module_utils.six.moves import map, reduce
from ansible.module_utils.pycompat24 import get_exception from ansible.module_utils.pycompat24 import get_exception
_NUMBERTYPES = tuple(list(integer_types) + [float])
# Deprecated compat. Only kept in case another module used these names Using
# ansible.module_utils.six is preferred
NUMBERTYPES = _NUMBERTYPES
imap = map
try:
# Python 2
unicode
except NameError:
# Python 3
unicode = text_type
try:
# Python 2.6+
bytes
except NameError:
# Python 2.4
bytes = binary_type
try:
# Python 2
basestring
except NameError:
# Python 3
basestring = string_types
# End of deprecated names
# Internal global holding passed in params. This is consulted in case # Internal global holding passed in params. This is consulted in case
# multiple AnsibleModules are created. Otherwise each AnsibleModule would # multiple AnsibleModules are created. Otherwise each AnsibleModule would
# attempt to read from stdin. Other code should not use this directly as it # attempt to read from stdin. Other code should not use this directly as it
@ -354,14 +338,14 @@ def json_dict_unicode_to_bytes(d, encoding='utf-8'):
and dict container types (the containers that the json module returns) and dict container types (the containers that the json module returns)
''' '''
if isinstance(d, unicode): if isinstance(d, text_type):
return d.encode(encoding) return d.encode(encoding)
elif isinstance(d, dict): elif isinstance(d, dict):
return dict(imap(json_dict_unicode_to_bytes, iteritems(d), repeat(encoding))) return dict(map(json_dict_unicode_to_bytes, iteritems(d), repeat(encoding)))
elif isinstance(d, list): elif isinstance(d, list):
return list(imap(json_dict_unicode_to_bytes, d, repeat(encoding))) return list(map(json_dict_unicode_to_bytes, d, repeat(encoding)))
elif isinstance(d, tuple): elif isinstance(d, tuple):
return tuple(imap(json_dict_unicode_to_bytes, d, repeat(encoding))) return tuple(map(json_dict_unicode_to_bytes, d, repeat(encoding)))
else: else:
return d return d
@ -372,28 +356,32 @@ def json_dict_bytes_to_unicode(d, encoding='utf-8'):
and dict container types (the containers that the json module returns) and dict container types (the containers that the json module returns)
''' '''
if isinstance(d, bytes): if isinstance(d, binary_type):
return unicode(d, encoding) # Warning, can traceback
return d.decode(encoding)
elif isinstance(d, dict): elif isinstance(d, dict):
return dict(imap(json_dict_bytes_to_unicode, iteritems(d), repeat(encoding))) return dict(map(json_dict_bytes_to_unicode, iteritems(d), repeat(encoding)))
elif isinstance(d, list): elif isinstance(d, list):
return list(imap(json_dict_bytes_to_unicode, d, repeat(encoding))) return list(map(json_dict_bytes_to_unicode, d, repeat(encoding)))
elif isinstance(d, tuple): elif isinstance(d, tuple):
return tuple(imap(json_dict_bytes_to_unicode, d, repeat(encoding))) return tuple(map(json_dict_bytes_to_unicode, d, repeat(encoding)))
else: else:
return d return d
def return_values(obj): def return_values(obj):
""" Return stringified values from datastructures. For use with removing """ Return native stringified values from datastructures.
sensitive values pre-jsonification."""
if isinstance(obj, basestring): For use with removing sensitive values pre-jsonification."""
if isinstance(obj, (text_type, binary_type)):
if obj: if obj:
if isinstance(obj, bytes): if isinstance(obj, text_type) and PY2:
yield obj
else:
# Unicode objects should all convert to utf-8 # Unicode objects should all convert to utf-8
# (still must deal with surrogateescape on python3)
yield obj.encode('utf-8') yield obj.encode('utf-8')
elif isinstance(obj, binary_type) and PY3:
yield obj.decode('utf-8', 'surrogateescape')
else:
# Already native string for this python version
yield obj
return return
elif isinstance(obj, SEQUENCETYPE): elif isinstance(obj, SEQUENCETYPE):
for element in obj: for element in obj:
@ -414,23 +402,29 @@ def return_values(obj):
def remove_values(value, no_log_strings): def remove_values(value, no_log_strings):
""" Remove strings in no_log_strings from value. If value is a container """ Remove strings in no_log_strings from value. If value is a container
type, then remove a lot more""" type, then remove a lot more"""
if isinstance(value, basestring): if isinstance(value, (text_type, binary_type)):
if isinstance(value, unicode): # Need native str type
# This should work everywhere on python2. Need to check native_str_value = value
# surrogateescape on python3 if isinstance(value, text_type):
bytes_value = value.encode('utf-8') value_is_text = True
value_is_unicode = True if PY2:
else: native_str_value = value.encode('utf-8')
bytes_value = value elif isinstance(value, binary_type):
value_is_unicode = False value_is_text = False
if bytes_value in no_log_strings: if PY3:
native_str_value = value.decode('utf-8', 'surrogateescape')
if native_str_value in no_log_strings:
return 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER' return 'VALUE_SPECIFIED_IN_NO_LOG_PARAMETER'
for omit_me in no_log_strings: for omit_me in no_log_strings:
bytes_value = bytes_value.replace(omit_me, '*' * 8) native_str_value = native_str_value.replace(omit_me, '*' * 8)
if value_is_unicode:
value = unicode(bytes_value, 'utf-8', errors='replace') if value_is_text and isinstance(native_str_value, binary_type):
value = native_str_value.decode('utf-8', 'replace')
elif not value_is_text and isinstance(native_str_value, text_type):
value = native_str_value.encode('utf-8', 'surrogateescape')
else: else:
value = bytes_value value = native_str_value
elif isinstance(value, SEQUENCETYPE): elif isinstance(value, SEQUENCETYPE):
return [remove_values(elem, no_log_strings) for elem in value] return [remove_values(elem, no_log_strings) for elem in value]
elif isinstance(value, Mapping): elif isinstance(value, Mapping):
@ -788,7 +782,7 @@ class AnsibleModule(object):
that your filesystem encoding is UTF-8. that your filesystem encoding is UTF-8.
''' '''
if isinstance(path, unicode): if isinstance(path, text_type):
path = path.encode("utf-8") path = path.encode("utf-8")
return path return path
@ -1380,25 +1374,25 @@ class AnsibleModule(object):
else: else:
self.fail_json(msg="internal error: choices for argument %s are not iterable: %s" % (k, choices)) self.fail_json(msg="internal error: choices for argument %s are not iterable: %s" % (k, choices))
def safe_eval(self, str, locals=None, include_exceptions=False): def safe_eval(self, value, locals=None, include_exceptions=False):
# do not allow method calls to modules # do not allow method calls to modules
if not isinstance(str, basestring): if not isinstance(value, string_types):
# already templated to a datastructure, perhaps? # already templated to a datavaluestructure, perhaps?
if include_exceptions: if include_exceptions:
return (str, None) return (value, None)
return str return value
if re.search(r'\w\.\w+\(', str): if re.search(r'\w\.\w+\(', value):
if include_exceptions: if include_exceptions:
return (str, None) return (value, None)
return str return value
# do not allow imports # do not allow imports
if re.search(r'import \w+', str): if re.search(r'import \w+', value):
if include_exceptions: if include_exceptions:
return (str, None) return (value, None)
return str return value
try: try:
result = literal_eval(str) result = literal_eval(value)
if include_exceptions: if include_exceptions:
return (result, None) return (result, None)
else: else:
@ -1406,11 +1400,11 @@ class AnsibleModule(object):
except Exception: except Exception:
e = get_exception() e = get_exception()
if include_exceptions: if include_exceptions:
return (str, e) return (value, e)
return str return value
def _check_type_str(self, value): def _check_type_str(self, value):
if isinstance(value, basestring): if isinstance(value, string_types):
return value return value
# Note: This could throw a unicode error if value's __str__() method # Note: This could throw a unicode error if value's __str__() method
# returns non-ascii. Have to port utils.to_bytes() if that happens # returns non-ascii. Have to port utils.to_bytes() if that happens
@ -1420,7 +1414,7 @@ class AnsibleModule(object):
if isinstance(value, list): if isinstance(value, list):
return value return value
if isinstance(value, basestring): if isinstance(value, string_types):
return value.split(",") return value.split(",")
elif isinstance(value, int) or isinstance(value, float): elif isinstance(value, int) or isinstance(value, float):
return [ str(value) ] return [ str(value) ]
@ -1431,7 +1425,7 @@ class AnsibleModule(object):
if isinstance(value, dict): if isinstance(value, dict):
return value return value
if isinstance(value, basestring): if isinstance(value, string_types):
if value.startswith("{"): if value.startswith("{"):
try: try:
return json.loads(value) return json.loads(value)
@ -1476,7 +1470,7 @@ class AnsibleModule(object):
if isinstance(value, bool): if isinstance(value, bool):
return value return value
if isinstance(value, basestring) or isinstance(value, int): if isinstance(value, string_types) or isinstance(value, int):
return self.boolean(value) return self.boolean(value)
raise TypeError('%s cannot be converted to a bool' % type(value)) raise TypeError('%s cannot be converted to a bool' % type(value))
@ -1485,7 +1479,7 @@ class AnsibleModule(object):
if isinstance(value, int): if isinstance(value, int):
return value return value
if isinstance(value, basestring): if isinstance(value, string_types):
return int(value) return int(value)
raise TypeError('%s cannot be converted to an int' % type(value)) raise TypeError('%s cannot be converted to an int' % type(value))
@ -1494,7 +1488,7 @@ class AnsibleModule(object):
if isinstance(value, float): if isinstance(value, float):
return value return value
if isinstance(value, basestring): if isinstance(value, string_types):
return float(value) return float(value)
raise TypeError('%s cannot be converted to a float' % type(value)) raise TypeError('%s cannot be converted to a float' % type(value))
@ -1506,7 +1500,7 @@ class AnsibleModule(object):
def _check_type_jsonarg(self, value): def _check_type_jsonarg(self, value):
# Return a jsonified string. Sometimes the controller turns a json # Return a jsonified string. Sometimes the controller turns a json
# string into a dict/list so transform it back into json here # string into a dict/list so transform it back into json here
if isinstance(value, (unicode, bytes)): if isinstance(value, (text_type, binary_type)):
return value.strip() return value.strip()
else: else:
if isinstance(value, (list, tuple, dict)): if isinstance(value, (list, tuple, dict)):
@ -1614,16 +1608,16 @@ class AnsibleModule(object):
log_args = dict() log_args = dict()
module = 'ansible-%s' % self._name module = 'ansible-%s' % self._name
if isinstance(module, bytes): if isinstance(module, binary_type):
module = module.decode('utf-8', 'replace') module = module.decode('utf-8', 'replace')
# 6655 - allow for accented characters # 6655 - allow for accented characters
if not isinstance(msg, (bytes, unicode)): if not isinstance(msg, (binary_type, text_type)):
raise TypeError("msg should be a string (got %s)" % type(msg)) raise TypeError("msg should be a string (got %s)" % type(msg))
# We want journal to always take text type # We want journal to always take text type
# syslog takes bytes on py2, text type on py3 # syslog takes bytes on py2, text type on py3
if isinstance(msg, bytes): if isinstance(msg, binary_type):
journal_msg = remove_values(msg.decode('utf-8', 'replace'), self.no_log_values) journal_msg = remove_values(msg.decode('utf-8', 'replace'), self.no_log_values)
else: else:
# TODO: surrogateescape is a danger here on Py3 # TODO: surrogateescape is a danger here on Py3
@ -1664,18 +1658,18 @@ class AnsibleModule(object):
log_args[param] = 'NOT_LOGGING_PASSWORD' log_args[param] = 'NOT_LOGGING_PASSWORD'
else: else:
param_val = self.params[param] param_val = self.params[param]
if not isinstance(param_val, basestring): if not isinstance(param_val, (text_type, binary_type)):
param_val = str(param_val) param_val = str(param_val)
elif isinstance(param_val, unicode): elif isinstance(param_val, text_type):
param_val = param_val.encode('utf-8') param_val = param_val.encode('utf-8')
log_args[param] = heuristic_log_sanitize(param_val, self.no_log_values) log_args[param] = heuristic_log_sanitize(param_val, self.no_log_values)
msg = [] msg = []
for arg in log_args: for arg in log_args:
arg_val = log_args[arg] arg_val = log_args[arg]
if not isinstance(arg_val, basestring): if not isinstance(arg_val, (text_type, binary_type)):
arg_val = str(arg_val) arg_val = str(arg_val)
elif isinstance(arg_val, unicode): elif isinstance(arg_val, text_type):
arg_val = arg_val.encode('utf-8') arg_val = arg_val.encode('utf-8')
msg.append('%s=%s' % (arg, arg_val)) msg.append('%s=%s' % (arg, arg_val))
if msg: if msg:
@ -1740,7 +1734,7 @@ class AnsibleModule(object):
''' return a bool for the arg ''' ''' return a bool for the arg '''
if arg is None or type(arg) == bool: if arg is None or type(arg) == bool:
return arg return arg
if isinstance(arg, basestring): if isinstance(arg, string_types):
arg = arg.lower() arg = arg.lower()
if arg in BOOLEANS_TRUE: if arg in BOOLEANS_TRUE:
return True return True
@ -2027,7 +2021,7 @@ class AnsibleModule(object):
if use_unsafe_shell: if use_unsafe_shell:
args = " ".join([pipes.quote(x) for x in args]) args = " ".join([pipes.quote(x) for x in args])
shell = True shell = True
elif isinstance(args, basestring) and use_unsafe_shell: elif isinstance(args, (binary_type, text_type)) and use_unsafe_shell:
shell = True shell = True
elif isinstance(args, string_types): elif isinstance(args, string_types):
# On python2.6 and below, shlex has problems with text type # On python2.6 and below, shlex has problems with text type
@ -2241,13 +2235,13 @@ class AnsibleModule(object):
def bytes_to_human(self, size): def bytes_to_human(self, size):
ranges = ( ranges = (
(1<<70, 'ZB'), (1 << 70, 'ZB'),
(1<<60, 'EB'), (1 << 60, 'EB'),
(1<<50, 'PB'), (1 << 50, 'PB'),
(1<<40, 'TB'), (1 << 40, 'TB'),
(1<<30, 'GB'), (1 << 30, 'GB'),
(1<<20, 'MB'), (1 << 20, 'MB'),
(1<<10, 'KB'), (1 << 10, 'KB'),
(1, 'Bytes') (1, 'Bytes')
) )
for limit, suffix in ranges: for limit, suffix in ranges:
@ -2265,7 +2259,7 @@ class AnsibleModule(object):
full = 'Bytes' full = 'Bytes'
if bits: if bits:
suffixes = [ x.replace('B', 'b') for x in suffixes ] suffixes = [ x.replace('B', 'b') for x in suffixes ]
full = 'Bits' full = 'Bits'
if number is None: if number is None: