From 07a4079a81927e0d4d9ae5a79ca0876505039f9c Mon Sep 17 00:00:00 2001 From: mikedlr Date: Tue, 11 Jul 2017 22:01:35 +0100 Subject: [PATCH] aws.core in new aws dir in module utils - module with AnsibleAWSModule class and fail_json_aws (#25780) * aws module utils including AnsibleAWSModule * fail_json_aws method on AnsibleAWSModule to do fail_json nicely with AWS exceptions * aws module util - feedback - rename to aws/core.py & improve doc strings --- lib/ansible/module_utils/aws/__init__.py | 0 lib/ansible/module_utils/aws/core.py | 143 ++++++++++++++++++ .../units/module_utils/aws/test_aws_module.py | 139 +++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 lib/ansible/module_utils/aws/__init__.py create mode 100644 lib/ansible/module_utils/aws/core.py create mode 100644 test/units/module_utils/aws/test_aws_module.py diff --git a/lib/ansible/module_utils/aws/__init__.py b/lib/ansible/module_utils/aws/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/ansible/module_utils/aws/core.py b/lib/ansible/module_utils/aws/core.py new file mode 100644 index 0000000000..57197f44e5 --- /dev/null +++ b/lib/ansible/module_utils/aws/core.py @@ -0,0 +1,143 @@ +# +# Copyright 2017 Michael De La Rue | Ansible +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + + +"""This module adds shared support for generic Amazon AWS modules + +**This code is not yet ready for use in user modules. As of 2017** +**and through to 2018, the interface is likely to change** +**aggressively as the exact correct interface for ansible AWS modules** +**is identified. In particular, until this notice goes away or is** +**changed, methods may disappear from the interface. Please don't** +**publish modules using this except directly to the main Ansible** +**development repository.** + +In order to use this module, include it as part of a custom +module as shown below. + + from ansible.module_utils.aws import AnsibleAWSModule + module = AnsibleAWSModule(argument_spec=dictionary, supports_check_mode=boolean + mutually_exclusive=list1, required_together=list2) + +The 'AnsibleAWSModule' module provides similar, but more restricted, +interfaces to the normal Ansible module. It also includes the +additional methods for connecting to AWS using the standard module arguments + + try: + m.aws_connect(resource='lambda') # - get an AWS connection. + except Exception: + m.fail_json_aws(Exception, msg="trying to connect") # - take an exception and make a decent failure + + +""" + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils._text import to_native +from ansible.module_utils.ec2 import HAS_BOTO3, camel_dict_to_snake_dict, ec2_argument_spec +import traceback + +# We will also export HAS_BOTO3 so end user modules can use it. +HAS_BOTO3 = HAS_BOTO3 + + +class AnsibleAWSModule(object): + """An ansible module class for AWS modules + + AnsibleAWSModule provides an a class for building modules which + connect to Amazon Web Services. The interface is currently more + restricted than the basic module class with the aim that later the + basic module class can be reduced. If you find that any key + feature is missing please contact the author/Ansible AWS team + (available on #ansible-aws on IRC) to request the additional + features needed. + """ + default_settings = { + "default_args": True, + "check_boto3": True, + "auto_retry": True, + "module_class": AnsibleModule + } + + def __init__(self, **kwargs): + local_settings = {} + for key in AnsibleAWSModule.default_settings: + try: + local_settings[key] = kwargs.pop(key) + except KeyError: + local_settings[key] = AnsibleAWSModule.default_settings[key] + self.settings = local_settings + + if local_settings["default_args"]: + # ec2_argument_spec contains the region so we use that; there's a patch coming which + # will add it to aws_argument_spec so if that's accepted then later we should change + # over + argument_spec_full = ec2_argument_spec() + try: + argument_spec_full.update(kwargs["argument_spec"]) + except (TypeError, NameError): + pass + kwargs["argument_spec"] = argument_spec_full + + self._module = AnsibleAWSModule.default_settings["module_class"](**kwargs) + + if local_settings["check_boto3"] and not HAS_BOTO3: + self._module.fail_json( + msg='Python modules "botocore" or "boto3" are missing, please install both') + + self.check_mode = self._module.check_mode + + @property + def params(self): + return self._module.params + + def exit_json(self, *args, **kwargs): + return self._module.exit_json(*args, **kwargs) + + def fail_json(self, *args, **kwargs): + return self._module.fail_json(*args, **kwargs) + + def fail_json_aws(self, exception, msg=None): + """call fail_json with processed exception + + function for converting exceptions thrown by AWS SDK modules, + botocore, boto3 and boto, into nice error messages. + """ + last_traceback = traceback.format_exc() + + # to_native is trusted to handle exceptions that str() could + # convert to text. + try: + except_msg = to_native(exception.message) + except AttributeError: + except_msg = to_native(exception) + + if msg is not None: + message = '{0}: {1}'.format(msg, except_msg) + else: + message = except_msg + + try: + response = exception.response + except AttributeError: + response = None + + if response is None: + self._module.fail_json(msg=message, exception=last_traceback) + else: + self._module.fail_json(msg=message, exception=last_traceback, + **camel_dict_to_snake_dict(response)) diff --git a/test/units/module_utils/aws/test_aws_module.py b/test/units/module_utils/aws/test_aws_module.py new file mode 100644 index 0000000000..e17deba70f --- /dev/null +++ b/test/units/module_utils/aws/test_aws_module.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# (c) 2017, Michael De La Rue +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +from pytest import importorskip +import unittest +from ansible.module_utils import basic +from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils._text import to_bytes +from ansible.compat.tests.mock import Mock, patch +import json + +importorskip("boto3") +botocore = importorskip("botocore") + + +class AWSModuleTestCase(unittest.TestCase): + + basic._ANSIBLE_ARGS = to_bytes(json.dumps({'ANSIBLE_MODULE_ARGS': {}})) + + def test_create_aws_module_should_set_up_params(self): + m = AnsibleAWSModule(argument_spec=dict( + win_string_arg=dict(type='list', default=['win']) + )) + m_noretry_no_customargs = AnsibleAWSModule( + auto_retry=False, default_args=False, + argument_spec=dict( + success_string_arg=dict(type='list', default=['success']) + ) + ) + assert m, "module wasn't true!!" + assert m_noretry_no_customargs, "module wasn't true!!" + + m_params = m.params + m_no_defs_params = m_noretry_no_customargs.params + assert 'region' in m_params + assert 'win' in m_params["win_string_arg"] + assert 'success' in m_no_defs_params["success_string_arg"] + assert 'aws_secret_key' not in m_no_defs_params + + +class ErrorReportingTestcase(unittest.TestCase): + + def test_botocore_exception_reports_nicely_via_fail_json_aws(self): + + basic._ANSIBLE_ARGS = to_bytes(json.dumps({'ANSIBLE_MODULE_ARGS': {}})) + module = AnsibleAWSModule(argument_spec=dict( + fail_mode=dict(type='list', default=['success']) + )) + + fail_json_double = Mock() + err_msg = {'Error': {'Code': 'FakeClass.FakeError'}} + with patch.object(basic.AnsibleModule, 'fail_json', fail_json_double): + try: + raise botocore.exceptions.ClientError(err_msg, 'Could not find you') + except Exception as e: + print("exception is " + str(e)) + module.fail_json_aws(e, msg="Fake failure for testing boto exception messages") + + assert(len(fail_json_double.mock_calls) > + 0), "failed to call fail_json when should have" + assert(len(fail_json_double.mock_calls) < + 2), "called fail_json multiple times when once would do" + assert("test_botocore_exception_reports_nicely" + in fail_json_double.mock_calls[0][2]["exception"]), \ + "exception traceback doesn't include correct function, fail call was actually: " \ + + str(fail_json_double.mock_calls[0]) + + assert("Fake failure for testing boto exception messages:" + in fail_json_double.mock_calls[0][2]["msg"]), \ + "error message doesn't include the local message; was: " \ + + str(fail_json_double.mock_calls[0]) + assert("Could not find you" in fail_json_double.mock_calls[0][2]["msg"]), \ + "error message doesn't include the botocore exception message; was: " \ + + str(fail_json_double.mock_calls[0]) + try: + fail_json_double.mock_calls[0][2]["error"] + except KeyError: + raise Exception("error was missing; call was: " + str(fail_json_double.mock_calls[0])) + assert("FakeClass.FakeError" == fail_json_double.mock_calls[0][2]["error"]["code"]), \ + "Failed to find error/code; was: " + str(fail_json_double.mock_calls[0]) + + def test_botocore_exception_without_response_reports_nicely_via_fail_json_aws(self): + basic._ANSIBLE_ARGS = to_bytes(json.dumps({'ANSIBLE_MODULE_ARGS': {}})) + module = AnsibleAWSModule(argument_spec=dict( + fail_mode=dict(type='list', default=['success']) + )) + + fail_json_double = Mock() + err_msg = None + with patch.object(basic.AnsibleModule, 'fail_json', fail_json_double): + try: + raise botocore.exceptions.ClientError(err_msg, 'Could not find you') + except Exception as e: + print("exception is " + str(e)) + module.fail_json_aws(e, msg="Fake failure for testing boto exception messages") + + assert(len(fail_json_double.mock_calls) > 0), "failed to call fail_json when should have" + assert(len(fail_json_double.mock_calls) < 2), "called fail_json multiple times" + + assert("test_botocore_exception_without_response_reports_nicely_via_fail_json_aws" + in fail_json_double.mock_calls[0][2]["exception"]), \ + "exception traceback doesn't include correct function, fail call was actually: " \ + + str(fail_json_double.mock_calls[0]) + + assert("Fake failure for testing boto exception messages" + in fail_json_double.mock_calls[0][2]["msg"]), \ + "error message doesn't include the local message; was: " \ + + str(fail_json_double.mock_calls[0]) + + # I would have thought this should work, however the botocore exception comes back with + # "argument of type 'NoneType' is not iterable" so it's probably not really designed + # to handle "None" as an error response. + # + # assert("Could not find you" in fail_json_double.mock_calls[0][2]["msg"]), \ + # "error message doesn't include the botocore exception message; was: " \ + # + str(fail_json_double.mock_calls[0]) + + +# TODO: +# - an exception without a message +# - plain boto exception +# - socket errors and other standard things.