From ebd08d2a01d96d41f8bc1ccf27bc1bbf91060a44 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 21 Nov 2017 16:41:27 -0500 Subject: [PATCH] jsonify inventory (#32990) * jsonify inventory * smarter import, dont pass kwargs where not needed * added datetime * Eventual plan for json utilities to migrate to common/json_utils when we split basic.py no need to move jsonify to another file now as we'll do that later. * json_dict_bytes_to_unicode and json_dict_unicode_to_bytes will also change names and move to common/text.py at that time (not to json). Their purpose is to recursively change the elements of a container (dict, list, set, tuple) into text or bytes, not to json encode or decode (they could be a generic precursor to that but are not limited to that.) * Reimplement the private _SetEncoder which changes sets and datetimes into objects that are json serializable into a private function instead. Functions are more flexible, less overhead, and simpler than an object. * Remove code that handled simplejson-1.5.x and earlier. Raise an error if that's the case instead. * We require python-2.6 or better which has the json module builtin to the stdlib. So this is only an issue if the stdlib json has been overridden by a third party module and the simplejson on the system is 1.5.x or less. (1.5 was released on 2007-01-18) --- lib/ansible/cli/inventory.py | 4 +- lib/ansible/module_utils/basic.py | 55 ++++++++++++++++---------- lib/ansible/module_utils/json_utils.py | 1 - test/units/module_utils/test_basic.py | 4 +- 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/lib/ansible/cli/inventory.py b/lib/ansible/cli/inventory.py index a9c08836e9..d839567dd8 100644 --- a/lib/ansible/cli/inventory.py +++ b/lib/ansible/cli/inventory.py @@ -178,8 +178,8 @@ class InventoryCLI(CLI): from ansible.parsing.yaml.dumper import AnsibleDumper results = yaml.dump(stuff, Dumper=AnsibleDumper, default_flow_style=False) else: - import json - results = json.dumps(stuff, sort_keys=True, indent=4) + from ansible.module_utils.basic import jsonify + results = jsonify(stuff, sort_keys=True, indent=4) return results diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index dfdd2df7d5..b469a95c9a 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -61,7 +61,7 @@ import errno import datetime from collections import deque from collections import Mapping, MutableMapping, Sequence, MutableSequence, Set, MutableSet -from itertools import repeat, chain +from itertools import chain, repeat try: import syslog @@ -111,6 +111,11 @@ except ImportError: except SyntaxError: print('\n{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}') sys.exit(1) + else: + sj_version = json.__version__.split('.') + if sj_version < ['1', '6']: + # Version 1.5 released 2007-01-18 does not have the encoding parameter which we need + print('\n{"msg": "Error: Ansible requires the stdlib json or simplejson >= 1.6. Neither was found!", "failed": true}') AVAILABLE_HASH_ALGORITHMS = dict() try: @@ -736,17 +741,34 @@ def get_flags_from_attributes(attributes): return ''.join(flags) +def _json_encode_fallback(obj): + if isinstance(obj, Set): + return list(obj) + elif isinstance(obj, datetime.datetime): + return obj.isoformat() + raise TypeError("Cannot json serialize %s" % to_native(obj)) + + +def jsonify(data, **kwargs): + for encoding in ("utf-8", "latin-1"): + try: + return json.dumps(data, encoding=encoding, default=_json_encode_fallback, **kwargs) + # Old systems using old simplejson module does not support encoding keyword. + except TypeError: + try: + new_data = json_dict_bytes_to_unicode(data, encoding=encoding) + except UnicodeDecodeError: + continue + return json.dumps(new_data, default=_json_encode_fallback, **kwargs) + except UnicodeDecodeError: + continue + raise UnicodeError('Invalid unicode encoding encountered') + + class AnsibleFallbackNotFound(Exception): pass -class _SetEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, Set): - return list(obj) - return super(_SetEncoder, self).default(obj) - - class AnsibleModule(object): def __init__(self, argument_spec, bypass_checks=False, no_log=False, check_invalid_arguments=True, mutually_exclusive=None, required_together=None, @@ -2180,19 +2202,10 @@ class AnsibleModule(object): self.fail_json(msg=to_native(e)) def jsonify(self, data): - for encoding in ("utf-8", "latin-1"): - try: - return json.dumps(data, encoding=encoding, cls=_SetEncoder) - # Old systems using old simplejson module does not support encoding keyword. - except TypeError: - try: - new_data = json_dict_bytes_to_unicode(data, encoding=encoding) - except UnicodeDecodeError: - continue - return json.dumps(new_data, cls=_SetEncoder) - except UnicodeDecodeError: - continue - self.fail_json(msg='Invalid unicode encoding encountered') + try: + return jsonify(data) + except UnicodeError as e: + self.fail_json(msg=to_text(e)) def from_json(self, data): return json.loads(data) diff --git a/lib/ansible/module_utils/json_utils.py b/lib/ansible/module_utils/json_utils.py index 0b190ed26a..181e955d54 100644 --- a/lib/ansible/module_utils/json_utils.py +++ b/lib/ansible/module_utils/json_utils.py @@ -23,7 +23,6 @@ # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - try: import json except ImportError: diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index 8298697f4c..4df155bc08 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -85,7 +85,9 @@ class TestModuleUtilsBasic(ModuleTestCase): if name == 'json': raise ImportError elif name == 'simplejson': - return MagicMock() + sj = MagicMock() + sj.__version__ = '3.10.0' + return sj return realimport(name, *args, **kwargs) self.clear_modules(['json', 'ansible.module_utils.basic'])