From e28845018d0f283249119221e60f3a980d3e3c89 Mon Sep 17 00:00:00 2001 From: Michael De La Rue Date: Mon, 5 Jun 2017 15:25:56 +0100 Subject: [PATCH] [cloud] New module: AWS API Gageway module (#20230) * Ultra basic api-gateway module based of lambda.py * Ultra basic deployment added to api-gateway module * ApiGateway module Allow creation of APIs, more documentation and better return value * ApiGateway module incorporate review feedback * ApiGateway module flake8 cleanup * APIGateway module - more review fixes. * slightly better messages in api_gateway module * AWS api_gateway module - try to improve messages in case of exceptions * rename api_gateway module to aws_api_gateway as discussed in PR 20230 * aws_api_gateway - Allow delivery of swagger either as text or dictionary. * aws_api_gateway module - introduce 'unit' tests, improve imports using them and small fixes * aws_api_gateway module - move path expand_user to avoid early typecheck * aws_api_gateway - version means version of metadata not module - fix to 1.0 * aws_api_gateway module - Rely on module_utils.ec2 for imports & path type for expanduser / cleanups * aws_api_gateway module - heavy cleanup and refactor of code + cloud retry functionality. * api_gateway_module - failing test case for handling more than one deployment in succession and API deletion * add TooManyRequestsException to AWSRetry exception list - makes API deployment work. * api_gateway_module - Fixes for various review comments + errors from various linters * api_gateway_module - Fixes for more review comments + linter error * api_gateway_module - Major refactor into sensible functions - create_response becomes configure_response * api_gateway_module - should be working under python3; remove test exclusion * api_gateway_module - finish off remaining review fixes - use ansible defaults and fix mutually exclusive * api_gateway_module - attempt to improve handling of botocore errors in python3 * api_gateway_module - implement state=absent / API deletion --- lib/ansible/module_utils/ec2.py | 10 +- .../modules/cloud/amazon/aws_api_gateway.py | 337 ++++++++++++++++++ .../targets/aws_api_gateway/aliases | 2 + .../targets/aws_api_gateway/meta/main.yml | 3 + .../targets/aws_api_gateway/tasks/main.yml | 182 ++++++++++ .../templates/minimal-swagger-api.yml.j2 | 33 ++ .../modules/cloud/amazon/test_api_gateway.py | 85 +++++ 7 files changed, 651 insertions(+), 1 deletion(-) create mode 100644 lib/ansible/modules/cloud/amazon/aws_api_gateway.py create mode 100644 test/integration/targets/aws_api_gateway/aliases create mode 100644 test/integration/targets/aws_api_gateway/meta/main.yml create mode 100644 test/integration/targets/aws_api_gateway/tasks/main.yml create mode 100644 test/integration/targets/aws_api_gateway/templates/minimal-swagger-api.yml.j2 create mode 100644 test/units/modules/cloud/amazon/test_api_gateway.py diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py index eb5db59f9c..93d9bee108 100644 --- a/lib/ansible/module_utils/ec2.py +++ b/lib/ansible/module_utils/ec2.py @@ -80,9 +80,17 @@ class AWSRetry(CloudRetry): def found(response_code): # This list of failures is based on this API Reference # http://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html + # + # TooManyRequestsException comes from inside botocore when it + # does retrys, unfortunately however it does not try long + # enough to allow some services such as API Gateway to + # complete configuration. At the moment of writing there is a + # botocore/boto3 bug open to fix this. + # + # https://github.com/boto/boto3/issues/876 (and linked PRs etc) retry_on = [ 'RequestLimitExceeded', 'Unavailable', 'ServiceUnavailable', - 'InternalFailure', 'InternalError' + 'InternalFailure', 'InternalError', 'TooManyRequestsException' ] not_found = re.compile(r'^\w+.NotFound') diff --git a/lib/ansible/modules/cloud/amazon/aws_api_gateway.py b/lib/ansible/modules/cloud/amazon/aws_api_gateway.py new file mode 100644 index 0000000000..cfe4aa2269 --- /dev/null +++ b/lib/ansible/modules/cloud/amazon/aws_api_gateway.py @@ -0,0 +1,337 @@ +#!/usr/bin/python +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + + +ANSIBLE_METADATA = {'status': ['preview'], + 'supported_by': 'community', + 'metadata_version': '1.0'} + +DOCUMENTATION = ''' +--- +module: aws_api_gateway +short_description: Manage AWS API Gateway APIs +description: + - Allows for the management of API Gatway APIs + - Normally you should give the api_id since there is no other + stable guaranteed unique identifier for the API. If you do + not give api_id then a new API will be create each time + this is run. + - Beware that there are very hard limits on the rate that + you can call API Gateway's REST API. You may need to patch + your boto. See https://github.com/boto/boto3/issues/876 + and discuss with your AWS rep. + - swagger_file and swagger_text are passed directly on to AWS + transparently whilst swagger_dict is an ansible dict which is + converted to JSON before the API definitions are uploaded. +version_added: '2.4' +requirements: [ boto3 ] +options: + api_id: + description: + - The ID of the API you want to manage. + state: + description: + - NOT IMPLEMENTED Create or delete API - currently we always create. + default: present + choices: [ 'present', 'absent' ] + swagger_file: + description: + - JSON or YAML file containing swagger definitions for API. + Exactly one of swagger_file, swagger_text or swagger_dict must + be present. + swagger_text: + description: + - Swagger definitions for API in JSON or YAML as a string direct + from playbook. + swagger_dict: + description: + - Swagger definitions API ansible dictionary which will be + converted to JSON and uploaded. + stage: + description: + - The name of the stage the API should be deployed to. + deploy_desc: + description: + - Description of the deployment - recorded and visible in the + AWS console. + default: Automatic deployment by Ansible. +author: + - 'Michael De La Rue (@mikedlr)' +extends_documentation_fragment: + - aws +notes: + - A future version of this module will probably use tags or another + ID so that an API can be create only once. + - As an early work around an intermediate version will probably do + the same using a tag embedded in the API name. + +''' + +EXAMPLES = ''' +# Update API resources for development +tasks: +- name: update API + aws_api_gateway: + api_id: 'abc123321cba' + state: present + swagger_file: my_api.yml + +# update definitions and deploy API to production +tasks: +- name: deploy API + aws_api_gateway: + api_id: 'abc123321cba' + state: present + swagger_file: my_api.yml + stage: production + deploy_desc: Make auth fix available. +''' + +RETURN = ''' +output: + description: the data returned by put_restapi in boto3 + returned: success + type: dict + sample: + 'data': + { + "id": "abc123321cba", + "name": "MY REST API", + "createdDate": 1484233401 + } +''' + +import json +from ansible.module_utils.basic import AnsibleModule, traceback +from ansible.module_utils.ec2 import ec2_argument_spec, get_aws_connection_info, boto3_conn, camel_dict_to_snake_dict, AWSRetry + +from ansible.module_utils.ec2 import HAS_BOTO3 + +try: + import botocore + HAS_BOTOCORE = True +except ImportError: + HAS_BOTOCORE = False + + +def main(): + argument_spec = ec2_argument_spec() + argument_spec.update( + dict( + api_id=dict(type='str', required=False), + state=dict(type='str', default='present', choices=['present', 'absent']), + swagger_file=dict(type='path', default=None, aliases=['src', 'api_file']), + swagger_dict=dict(type='json', default=None), + swagger_text=dict(type='str', default=None), + stage=dict(type='str', default=None), + deploy_desc=dict(type='str', default="Automatic deployment by Ansible."), + ) + ) + + mutually_exclusive = [['swagger_file', 'swagger_dict', 'swagger_text']] # noqa: F841 + + module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False, + mutually_exclusive=mutually_exclusive) + + api_id = module.params.get('api_id') + state = module.params.get('state') # noqa: F841 + swagger_file = module.params.get('swagger_file') + swagger_dict = module.params.get('swagger_dict') + swagger_text = module.params.get('swagger_text') + stage = module.params.get('stage') + deploy_desc = module.params.get('deploy_desc') + +# check_mode = module.check_mode + changed = False + + if not HAS_BOTO3: + module.fail_json(msg='Python module "boto3" is missing, please install boto3') + + if not HAS_BOTOCORE: + module.fail_json(msg='Python module "botocore" is missing, please install it') + + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) + try: + client = boto3_conn(module, conn_type='client', resource='apigateway', + region=region, endpoint=ec2_url, **aws_connect_kwargs) + except botocore.exceptions.NoRegionError: + module.fail_json(msg="Region must be specified as a parameter, in " + "AWS_DEFAULT_REGION environment variable or in boto configuration file") + except (botocore.exceptions.ValidationError, botocore.exceptions.ClientError) as e: + fail_json_aws(module, e, msg="connecting to AWS") + + changed = True # for now it will stay that way until we can sometimes avoid change + + conf_res = None + dep_res = None + del_res = None + + if state == "present": + if api_id is None: + api_id = create_empty_api(module, client) + api_data = get_api_definitions(module, swagger_file=swagger_file, + swagger_dict=swagger_dict, swagger_text=swagger_text) + conf_res, dep_res = ensure_api_in_correct_state(module, client, api_id=api_id, + api_data=api_data, stage=stage, + deploy_desc=deploy_desc) + if state == "absent": + del_res = delete_rest_api(module, client, api_id) + + exit_args = {"changed": changed, "api_id": api_id} + + if conf_res is not None: + exit_args['configure_response'] = camel_dict_to_snake_dict(conf_res) + if dep_res is not None: + exit_args['deploy_response'] = camel_dict_to_snake_dict(dep_res) + if del_res is not None: + exit_args['delete_response'] = camel_dict_to_snake_dict(del_res) + + module.exit_json(**exit_args) + + +def get_api_definitions(module, swagger_file=None, swagger_dict=None, swagger_text=None): + apidata = None + if swagger_file is not None: + try: + with open(swagger_file) as f: + apidata = f.read() + except OSError as e: + msg = "Failed trying to read swagger file {}: {}".format(str(swagger_file), str(e)) + module.fail_json(msg=msg, exception=traceback.format_exc()) + if swagger_dict is not None: + apidata = json.dumps(swagger_dict) + if swagger_text is not None: + apidata = swagger_text + + if apidata is None: + module.fail_json(msg='module error - failed to get API data') + return apidata + + +def create_empty_api(module, client): + """ + creates a new empty API ready to be configured. The description is + temporarily set to show the API as incomplete but should be + updated when the API is configured. + """ + desc = "Incomplete API creation by ansible aws_api_gateway module" + try: + awsret = create_api(client, name="ansible-temp-api", description=desc) + except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e: + fail_json_aws(module, e, msg="creating API") + return awsret["id"] + + +def delete_rest_api(module, client, api_id): + """ + creates a new empty API ready to be configured. The description is + temporarily set to show the API as incomplete but should be + updated when the API is configured. + """ + try: + delete_response = delete_api(client, api_id=api_id) + except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e: + fail_json_aws(module, e, msg="deleting API {}".format(api_id)) + return delete_response + + +def ensure_api_in_correct_state(module, client, api_id=None, api_data=None, stage=None, + deploy_desc=None): + """Make sure that we have the API configured and deployed as instructed. + + This function first configures the API correctly uploading the + swagger definitions and then deploys those. Configuration and + deployment should be closely tied because there is only one set of + definitions so if we stop, they may be updated by someone else and + then we deploy the wrong configuration. + """ + + configure_response = None + try: + configure_response = configure_api(client, api_data=api_data, api_id=api_id) + except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e: + fail_json_aws(module, e, msg="configuring API {}".format(api_id)) + + deploy_response = None + + if stage: + try: + deploy_response = create_deployment(client, api_id=api_id, stage=stage, + description=deploy_desc) + except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e: + msg = "deploying api {} to stage {}".format(api_id, stage) + fail_json_aws(module, e, msg) + + return configure_response, deploy_response + + +# There is a PR open to merge fail_json_aws this into the standard module code; +# see https://github.com/ansible/ansible/pull/23882 +def fail_json_aws(module, exception, msg=None): + """call fail_json with processed exception + function for converting exceptions thrown by AWS SDK modules, + botocore, boto3 and boto, into nice error messages. + """ + last_traceback = traceback.format_exc() + + try: + except_msg = exception.message + except AttributeError: + except_msg = str(exception) + + if msg is not None: + message = '{}: {}'.format(msg, except_msg) + else: + message = except_msg + + try: + response = exception.response + except AttributeError: + response = None + + if response is None: + module.fail_json(msg=message, traceback=last_traceback) + else: + module.fail_json(msg=message, traceback=last_traceback, + **camel_dict_to_snake_dict(response)) + + +retry_params = {"tries": 10, "delay": 5, "backoff": 1.2} + + +@AWSRetry.backoff(**retry_params) +def create_api(client, name=None, description=None): + return client.create_rest_api(name="ansible-temp-api", description=description) + + +@AWSRetry.backoff(**retry_params) +def delete_api(client, api_id=None): + return client.delete_rest_api(restApiId=api_id) + + +@AWSRetry.backoff(**retry_params) +def configure_api(client, api_data=None, api_id=None, mode="overwrite"): + return client.put_rest_api(body=api_data, restApiId=api_id, mode=mode) + + +@AWSRetry.backoff(**retry_params) +def create_deployment(client, api_id=None, stage=None, description=None): + # we can also get None as an argument so we don't do this as a defult + return client.create_deployment(restApiId=api_id, stageName=stage, description=description) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/aws_api_gateway/aliases b/test/integration/targets/aws_api_gateway/aliases new file mode 100644 index 0000000000..495c6e74ed --- /dev/null +++ b/test/integration/targets/aws_api_gateway/aliases @@ -0,0 +1,2 @@ +cloud/aws +posix/ci/cloud/aws diff --git a/test/integration/targets/aws_api_gateway/meta/main.yml b/test/integration/targets/aws_api_gateway/meta/main.yml new file mode 100644 index 0000000000..1f64f1169a --- /dev/null +++ b/test/integration/targets/aws_api_gateway/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_tests + - setup_ec2 diff --git a/test/integration/targets/aws_api_gateway/tasks/main.yml b/test/integration/targets/aws_api_gateway/tasks/main.yml new file mode 100644 index 0000000000..5b4c2691e2 --- /dev/null +++ b/test/integration/targets/aws_api_gateway/tasks/main.yml @@ -0,0 +1,182 @@ +- block: + + # ============================================================ + - name: test with no parameters + aws_api_gateway: + register: result + ignore_errors: true + + - name: assert failure when called with no parameters + assert: + that: + - 'result.failed' + - 'result.msg.startswith("Region must be specified")' + + # ============================================================ + - name: test with minimal parameters but no region + aws_api_gateway: + api_id: 'fake-api-doesnt-exist' + register: result + ignore_errors: true + + - name: assert failure when called with with minimal parameters but no region + assert: + that: + - 'result.failed' + - 'result.msg.startswith("Region must be specified")' + + # ============================================================ + - name: test disallow multiple swagger sources + aws_api_gateway: + api_id: 'fake-api-doesnt-exist' + region: 'fake_region' + swagger_file: foo.yml + swagger_text: "this is not really an API" + register: result + ignore_errors: true + + - name: assert failure when called with with minimal parameters but no region + assert: + that: + - 'result.failed' + - 'result.msg.startswith("parameters are mutually exclusive")' + + # This fails with + + # msg": "There is an issue in the code of the module. You must + # specify either both, resource or client to the conn_type + # parameter in the boto3_conn function call" + + # even though the call appears to include conn_type='client' + + # # ============================================================ + # - name: test invalid region parameter + # aws_api_gateway: + # api_id: 'fake-api-doesnt-exist' + # region: 'asdf querty 1234' + # register: result + # ignore_errors: true + + # - name: assert invalid region parameter + # assert: + # that: + # - 'result.failed' + # - 'result.msg.startswith("Region asdf querty 1234 does not seem to be available ")' + + # ============================================================ + + - name: build API file + template: + src: minimal-swagger-api.yml.j2 + dest: "{{output_dir}}/minimal-swagger-api.yml" + tags: new_api,api,api_file + + - name: deploy new API + aws_api_gateway: + api_file: "{{output_dir}}/minimal-swagger-api.yml" + stage: "minimal" + region: '{{ec2_region}}' + aws_access_key: '{{ec2_access_key}}' + aws_secret_key: '{{ec2_secret_key}}' + security_token: '{{security_token}}' + register: create_result + + - name: assert deploy new API worked + assert: + that: + - 'create_result.changed == True' + - '"api_id" in create_result' +# - '"created_response.created_date" in create_result' +# - '"deploy_response.created_date" in create_result' + + - name: check API works + uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/minimal" + register: uri_result + + - name: assert API works success + assert: + that: + - 'uri_result' + + - name: check nonexistent endpoints cause errors + uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/nominal" + register: bad_uri_result + ignore_errors: true + + - name: assert + assert: + that: + - bad_uri_result|failed + + # ============================================================ + + - name: deploy first API + aws_api_gateway: + api_file: "{{output_dir}}/minimal-swagger-api.yml" + stage: "minimal" + region: '{{ec2_region}}' + aws_access_key: '{{ec2_access_key}}' + aws_secret_key: '{{ec2_secret_key}}' + security_token: '{{security_token}}' + register: create_result_1 + + - name: deploy second API rapidly after first + aws_api_gateway: + api_file: "{{output_dir}}/minimal-swagger-api.yml" + stage: "minimal" + region: '{{ec2_region}}' + aws_access_key: '{{ec2_access_key}}' + aws_secret_key: '{{ec2_secret_key}}' + security_token: '{{security_token}}' + register: create_result_2 + + - name: assert both APIs deployed successfully + assert: + that: + - 'create_result_1.changed == True' + - 'create_result_2.changed == True' + - '"api_id" in create_result_1' + - '"api_id" in create_result_1' +# - '"created_response.created_date" in create_result' +# - '"deploy_response.created_date" in create_result' + + - name: destroy first API + aws_api_gateway: + state: absent + api_id: '{{create_result_1.api_id}}' + region: '{{ec2_region}}' + aws_access_key: '{{ec2_access_key}}' + aws_secret_key: '{{ec2_secret_key}}' + security_token: '{{security_token}}' + register: destroy_result_1 + + - name: destroy second API rapidly after first + aws_api_gateway: + state: absent + api_id: '{{create_result_2.api_id}}' + region: '{{ec2_region}}' + aws_access_key: '{{ec2_access_key}}' + aws_secret_key: '{{ec2_secret_key}}' + security_token: '{{security_token}}' + register: destroy_result_2 + + - name: assert both APIs deployed successfully + assert: + that: + - 'destroy_result_1.changed == True' + - 'destroy_result_2.changed == True' +# - '"created_response.created_date" in create_result' +# - '"deploy_response.created_date" in create_result' + + always: + + # ============================================================ + - name: test state=absent (expect changed=false) + aws_api_gateway: + state: absent + api_id: '{{create_result.api_id}}' + ec2_region: '{{ec2_region}}' + aws_access_key: '{{ec2_access_key}}' + aws_secret_key: '{{ec2_secret_key}}' + security_token: '{{security_token}}' + register: destroy_result diff --git a/test/integration/targets/aws_api_gateway/templates/minimal-swagger-api.yml.j2 b/test/integration/targets/aws_api_gateway/templates/minimal-swagger-api.yml.j2 new file mode 100644 index 0000000000..8c5c058106 --- /dev/null +++ b/test/integration/targets/aws_api_gateway/templates/minimal-swagger-api.yml.j2 @@ -0,0 +1,33 @@ +--- +swagger: "2.0" +info: + version: "2017-05-11T12:14:59Z" + title: "{{resource_prefix}}Empty_API" +host: "fakeexample.execute-api.us-east-1.amazonaws.com" +basePath: "/minimal" +schemes: +- "https" +paths: + /: + get: + consumes: + - "application/json" + produces: + - "application/json" + responses: + 200: + description: "200 response" + schema: + $ref: "#/definitions/Empty" + x-amazon-apigateway-integration: + responses: + default: + statusCode: "200" + requestTemplates: + application/json: "{\"statusCode\": 200}" + passthroughBehavior: "when_no_match" + type: "mock" +definitions: + Empty: + type: "object" + title: "Empty Schema" diff --git a/test/units/modules/cloud/amazon/test_api_gateway.py b/test/units/modules/cloud/amazon/test_api_gateway.py new file mode 100644 index 0000000000..f9a678ef70 --- /dev/null +++ b/test/units/modules/cloud/amazon/test_api_gateway.py @@ -0,0 +1,85 @@ +# +# (c) 2016 Michael De La Rue +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish + +from __future__ import (absolute_import, division, print_function) + +from nose.plugins.skip import SkipTest +import pytest +import sys +import json +from ansible.module_utils._text import to_bytes +from ansible.module_utils import basic +from ansible.module_utils.ec2 import HAS_BOTO3 + +if not HAS_BOTO3: + raise SkipTest("test_api_gateway.py requires the `boto3` and `botocore` modules") + +import ansible.modules.cloud.amazon.aws_api_gateway as agw + + +def set_module_args(args): + args = json.dumps({'ANSIBLE_MODULE_ARGS': args}) + basic._ANSIBLE_ARGS = to_bytes(args) + + +exit_return_dict = {} + + +def fake_exit_json(self, **kwargs): + """ store the kwargs given to exit_json rather than putting them out to stdout""" + global exit_return_dict + exit_return_dict = kwargs + sys.exit(0) + + +def test_upload_api(monkeypatch): + class FakeConnection: + + def put_rest_api(self, *args, **kwargs): + assert kwargs["body"] == "the-swagger-text-is-fake" + return {"msg": "success!"} + + def return_fake_connection(*args, **kwargs): + return FakeConnection() + + monkeypatch.setattr(agw, "boto3_conn", return_fake_connection) + monkeypatch.setattr(agw.AnsibleModule, "exit_json", fake_exit_json) + + set_module_args({ + "api_id": "fred", + "state": "present", + "swagger_text": "the-swagger-text-is-fake", + "region": 'mars-north-1', + }) + with pytest.raises(SystemExit): + agw.main() + assert exit_return_dict["changed"] + + +def test_warn_if_region_not_specified(): + + set_module_args({ + "name": "aws_api_gateway", + "state": "present", + "runtime": 'python2.7', + "role": 'arn:aws:iam::987654321012:role/lambda_basic_execution', + "handler": 'lambda_python.my_handler'}) + with pytest.raises(SystemExit): + print(agw.main())