[cloud] New module: AWS API Gageway module (#20230)

* Ultra basic api-gateway module based of lambda.py

* Ultra basic deployment added to api-gateway module

* ApiGateway module Allow creation of APIs, more documentation and better return value

* ApiGateway module incorporate review feedback

* ApiGateway module flake8 cleanup

* APIGateway module - more review fixes.

* slightly better messages in api_gateway module

* AWS api_gateway module - try to improve messages in case of exceptions

* rename api_gateway module to aws_api_gateway as discussed in PR 20230

* aws_api_gateway - Allow delivery of swagger either as text or dictionary.

* aws_api_gateway module - introduce 'unit' tests, improve imports using them and small fixes

* aws_api_gateway module - move path expand_user to avoid early typecheck

* aws_api_gateway - version means version of metadata not module - fix to 1.0

* aws_api_gateway module - Rely on module_utils.ec2 for imports & path type for expanduser / cleanups

* aws_api_gateway module - heavy cleanup and refactor of code + cloud retry functionality.

* api_gateway_module - failing test case for handling more than one deployment in succession and API deletion

* add TooManyRequestsException to AWSRetry exception list - makes API deployment work.

* api_gateway_module - Fixes for various review comments + errors from various linters

* api_gateway_module - Fixes for more review comments + linter error

* api_gateway_module - Major refactor into sensible functions - create_response becomes configure_response

* api_gateway_module - should be working under python3; remove test exclusion

* api_gateway_module - finish off remaining review fixes - use ansible defaults and fix mutually exclusive

* api_gateway_module - attempt to improve handling of botocore errors in python3

* api_gateway_module - implement state=absent / API deletion

lib/ansible/module_utils/ec2.py

@@ -80,9 +80,17 @@ class AWSRetry(CloudRetry):
     def found(response_code):
         # This list of failures is based on this API Reference
         # http://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html
+        #
+        # TooManyRequestsException comes from inside botocore when it
+        # does retrys, unfortunately however it does not try long
+        # enough to allow some services such as API Gateway to
+        # complete configuration.  At the moment of writing there is a
+        # botocore/boto3 bug open to fix this.
+        #
+        # https://github.com/boto/boto3/issues/876 (and linked PRs etc)
         retry_on = [
             'RequestLimitExceeded', 'Unavailable', 'ServiceUnavailable',
-            'InternalFailure', 'InternalError'
+            'InternalFailure', 'InternalError', 'TooManyRequestsException'
         ]
 
         not_found = re.compile(r'^\w+.NotFound')

lib/ansible/modules/cloud/amazon/aws_api_gateway.py

@@ -0,0 +1,337 @@
+#!/usr/bin/python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+ANSIBLE_METADATA = {'status': ['preview'],
+                    'supported_by': 'community',
+                    'metadata_version': '1.0'}
+
+DOCUMENTATION = '''
+---
+module: aws_api_gateway
+short_description: Manage AWS API Gateway APIs
+description:
+     - Allows for the management of API Gatway APIs
+     - Normally you should give the api_id since there is no other
+       stable guaranteed unique identifier for the API.  If you do
+       not give api_id then a new API will be create each time
+       this is run.
+     - Beware that there are very hard limits on the rate that
+       you can call API Gateway's REST API.  You may need to patch
+       your boto.  See https://github.com/boto/boto3/issues/876
+       and discuss with your AWS rep.
+     - swagger_file and swagger_text are passed directly on to AWS
+       transparently whilst swagger_dict is an ansible dict which is
+       converted to JSON before the API definitions are uploaded.
+version_added: '2.4'
+requirements: [ boto3 ]
+options:
+  api_id:
+    description:
+      - The ID of the API you want to manage.
+  state:
+    description:
+      - NOT IMPLEMENTED Create or delete API - currently we always create.
+    default: present
+    choices: [ 'present', 'absent' ]
+  swagger_file:
+    description:
+      - JSON or YAML file containing swagger definitions for API.
+        Exactly one of swagger_file, swagger_text or swagger_dict must
+        be present.
+  swagger_text:
+    description:
+      - Swagger definitions for API in JSON or YAML as a string direct
+        from playbook.
+  swagger_dict:
+    description:
+      - Swagger definitions API ansible dictionary which will be
+        converted to JSON and uploaded.
+  stage:
+    description:
+      - The name of the stage the API should be deployed to.
+  deploy_desc:
+    description:
+      - Description of the deployment - recorded and visible in the
+        AWS console.
+    default: Automatic deployment by Ansible.
+author:
+    - 'Michael De La Rue (@mikedlr)'
+extends_documentation_fragment:
+    - aws
+notes:
+   - A future version of this module will probably use tags or another
+     ID so that an API can be create only once.
+   - As an early work around an intermediate version will probably do
+     the same using a tag embedded in the API name.
+
+'''
+
+EXAMPLES = '''
+# Update API resources for development
+tasks:
+- name: update API
+  aws_api_gateway:
+    api_id: 'abc123321cba'
+    state: present
+    swagger_file: my_api.yml
+
+# update definitions and deploy API to production
+tasks:
+- name: deploy API
+  aws_api_gateway:
+    api_id: 'abc123321cba'
+    state: present
+    swagger_file: my_api.yml
+    stage: production
+    deploy_desc: Make auth fix available.
+'''
+
+RETURN = '''
+output:
+  description: the data returned by put_restapi in boto3
+  returned: success
+  type: dict
+  sample:
+    'data':
+      {
+          "id": "abc123321cba",
+          "name": "MY REST API",
+          "createdDate": 1484233401
+      }
+'''
+
+import json
+from ansible.module_utils.basic import AnsibleModule, traceback
+from ansible.module_utils.ec2 import ec2_argument_spec, get_aws_connection_info, boto3_conn, camel_dict_to_snake_dict, AWSRetry
+
+from ansible.module_utils.ec2 import HAS_BOTO3
+
+try:
+    import botocore
+    HAS_BOTOCORE = True
+except ImportError:
+    HAS_BOTOCORE = False
+
+
+def main():
+    argument_spec = ec2_argument_spec()
+    argument_spec.update(
+        dict(
+            api_id=dict(type='str', required=False),
+            state=dict(type='str', default='present', choices=['present', 'absent']),
+            swagger_file=dict(type='path', default=None, aliases=['src', 'api_file']),
+            swagger_dict=dict(type='json', default=None),
+            swagger_text=dict(type='str', default=None),
+            stage=dict(type='str', default=None),
+            deploy_desc=dict(type='str', default="Automatic deployment by Ansible."),
+        )
+    )
+
+    mutually_exclusive = [['swagger_file', 'swagger_dict', 'swagger_text']]  # noqa: F841
+
+    module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False,
+                           mutually_exclusive=mutually_exclusive)
+
+    api_id = module.params.get('api_id')
+    state = module.params.get('state')   # noqa: F841
+    swagger_file = module.params.get('swagger_file')
+    swagger_dict = module.params.get('swagger_dict')
+    swagger_text = module.params.get('swagger_text')
+    stage = module.params.get('stage')
+    deploy_desc = module.params.get('deploy_desc')
+
+#    check_mode = module.check_mode
+    changed = False
+
+    if not HAS_BOTO3:
+        module.fail_json(msg='Python module "boto3" is missing, please install boto3')
+
+    if not HAS_BOTOCORE:
+        module.fail_json(msg='Python module "botocore" is missing, please install it')
+
+    region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
+    try:
+        client = boto3_conn(module, conn_type='client', resource='apigateway',
+                            region=region, endpoint=ec2_url, **aws_connect_kwargs)
+    except botocore.exceptions.NoRegionError:
+        module.fail_json(msg="Region must be specified as a parameter, in "
+                         "AWS_DEFAULT_REGION environment variable or in boto configuration file")
+    except (botocore.exceptions.ValidationError, botocore.exceptions.ClientError) as e:
+        fail_json_aws(module, e, msg="connecting to AWS")
+
+    changed = True   # for now it will stay that way until we can sometimes avoid change
+
+    conf_res = None
+    dep_res = None
+    del_res = None
+
+    if state == "present":
+        if api_id is None:
+            api_id = create_empty_api(module, client)
+        api_data = get_api_definitions(module, swagger_file=swagger_file,
+                                       swagger_dict=swagger_dict, swagger_text=swagger_text)
+        conf_res, dep_res = ensure_api_in_correct_state(module, client, api_id=api_id,
+                                                        api_data=api_data, stage=stage,
+                                                        deploy_desc=deploy_desc)
+    if state == "absent":
+        del_res = delete_rest_api(module, client, api_id)
+
+    exit_args = {"changed": changed, "api_id": api_id}
+
+    if conf_res is not None:
+        exit_args['configure_response'] = camel_dict_to_snake_dict(conf_res)
+    if dep_res is not None:
+        exit_args['deploy_response'] = camel_dict_to_snake_dict(dep_res)
+    if del_res is not None:
+        exit_args['delete_response'] = camel_dict_to_snake_dict(del_res)
+
+    module.exit_json(**exit_args)
+
+
+def get_api_definitions(module, swagger_file=None, swagger_dict=None, swagger_text=None):
+    apidata = None
+    if swagger_file is not None:
+        try:
+            with open(swagger_file) as f:
+                apidata = f.read()
+        except OSError as e:
+            msg = "Failed trying to read swagger file {}: {}".format(str(swagger_file), str(e))
+            module.fail_json(msg=msg, exception=traceback.format_exc())
+    if swagger_dict is not None:
+        apidata = json.dumps(swagger_dict)
+    if swagger_text is not None:
+        apidata = swagger_text
+
+    if apidata is None:
+        module.fail_json(msg='module error - failed to get API data')
+    return apidata
+
+
+def create_empty_api(module, client):
+    """
+    creates a new empty API ready to be configured.  The description is
+    temporarily set to show the API as incomplete but should be
+    updated when the API is configured.
+    """
+    desc = "Incomplete API creation by ansible aws_api_gateway module"
+    try:
+        awsret = create_api(client, name="ansible-temp-api", description=desc)
+    except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
+        fail_json_aws(module, e, msg="creating API")
+    return awsret["id"]
+
+
+def delete_rest_api(module, client, api_id):
+    """
+    creates a new empty API ready to be configured.  The description is
+    temporarily set to show the API as incomplete but should be
+    updated when the API is configured.
+    """
+    try:
+        delete_response = delete_api(client, api_id=api_id)
+    except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
+        fail_json_aws(module, e, msg="deleting API {}".format(api_id))
+    return delete_response
+
+
+def ensure_api_in_correct_state(module, client, api_id=None, api_data=None, stage=None,
+                                deploy_desc=None):
+    """Make sure that we have the API configured and deployed as instructed.
+
+    This function first configures the API correctly uploading the
+    swagger definitions and then deploys those.  Configuration and
+    deployment should be closely tied because there is only one set of
+    definitions so if we stop, they may be updated by someone else and
+    then we deploy the wrong configuration.
+    """
+
+    configure_response = None
+    try:
+        configure_response = configure_api(client, api_data=api_data, api_id=api_id)
+    except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
+        fail_json_aws(module, e, msg="configuring API {}".format(api_id))
+
+    deploy_response = None
+
+    if stage:
+        try:
+            deploy_response = create_deployment(client, api_id=api_id, stage=stage,
+                                                description=deploy_desc)
+        except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
+            msg = "deploying api {} to stage {}".format(api_id, stage)
+            fail_json_aws(module, e, msg)
+
+    return configure_response, deploy_response
+
+
+# There is a PR open to merge fail_json_aws this into the standard module code;
+# see https://github.com/ansible/ansible/pull/23882
+def fail_json_aws(module, exception, msg=None):
+    """call fail_json with processed exception
+    function for converting exceptions thrown by AWS SDK modules,
+    botocore, boto3 and boto, into nice error messages.
+    """
+    last_traceback = traceback.format_exc()
+
+    try:
+        except_msg = exception.message
+    except AttributeError:
+        except_msg = str(exception)
+
+    if msg is not None:
+        message = '{}: {}'.format(msg, except_msg)
+    else:
+        message = except_msg
+
+    try:
+        response = exception.response
+    except AttributeError:
+        response = None
+
+    if response is None:
+        module.fail_json(msg=message, traceback=last_traceback)
+    else:
+        module.fail_json(msg=message, traceback=last_traceback,
+                         **camel_dict_to_snake_dict(response))
+
+
+retry_params = {"tries": 10, "delay": 5, "backoff": 1.2}
+
+
+@AWSRetry.backoff(**retry_params)
+def create_api(client, name=None, description=None):
+    return client.create_rest_api(name="ansible-temp-api", description=description)
+
+
+@AWSRetry.backoff(**retry_params)
+def delete_api(client, api_id=None):
+    return client.delete_rest_api(restApiId=api_id)
+
+
+@AWSRetry.backoff(**retry_params)
+def configure_api(client, api_data=None, api_id=None, mode="overwrite"):
+    return client.put_rest_api(body=api_data, restApiId=api_id, mode=mode)
+
+
+@AWSRetry.backoff(**retry_params)
+def create_deployment(client, api_id=None, stage=None, description=None):
+    # we can also get None as an argument so we don't do this as a defult
+    return client.create_deployment(restApiId=api_id, stageName=stage, description=description)
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/aws_api_gateway/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+posix/ci/cloud/aws

test/integration/targets/aws_api_gateway/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - prepare_tests
+  - setup_ec2

test/integration/targets/aws_api_gateway/tasks/main.yml

@@ -0,0 +1,182 @@
+- block:
+
+    # ============================================================
+    - name: test with no parameters
+      aws_api_gateway:
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with no parameters
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("Region must be specified")'
+
+    # ============================================================
+    - name: test with minimal parameters but no region
+      aws_api_gateway:
+        api_id: 'fake-api-doesnt-exist'
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with with minimal parameters but no region
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("Region must be specified")'
+
+    # ============================================================
+    - name: test disallow multiple swagger sources
+      aws_api_gateway:
+        api_id: 'fake-api-doesnt-exist'
+        region: 'fake_region'
+        swagger_file: foo.yml
+        swagger_text: "this is not really an API"
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with with minimal parameters but no region
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("parameters are mutually exclusive")'
+
+    # This fails with
+
+    # msg": "There is an issue in the code of the module. You must
+    # specify either both, resource or client to the conn_type
+    # parameter in the boto3_conn function call"
+
+    # even though the call appears to include conn_type='client'
+
+    # # ============================================================
+    # - name: test invalid region parameter
+    #   aws_api_gateway:
+    #     api_id: 'fake-api-doesnt-exist'
+    #     region: 'asdf querty 1234'
+    #   register: result
+    #   ignore_errors: true
+
+    # - name: assert invalid region parameter
+    #   assert:
+    #     that:
+    #        - 'result.failed'
+    #        - 'result.msg.startswith("Region asdf querty 1234 does not seem to be available ")'
+
+    # ============================================================
+
+    - name: build API file 
+      template:
+        src: minimal-swagger-api.yml.j2
+        dest: "{{output_dir}}/minimal-swagger-api.yml"
+      tags: new_api,api,api_file
+
+    - name: deploy new API
+      aws_api_gateway: 
+        api_file: "{{output_dir}}/minimal-swagger-api.yml"
+        stage: "minimal"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result
+
+    - name: assert deploy new API worked
+      assert:
+        that:
+           - 'create_result.changed == True'
+           - '"api_id" in create_result'
+#           - '"created_response.created_date" in create_result'
+#           - '"deploy_response.created_date" in create_result'
+
+    - name: check API works
+      uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/minimal"
+      register: uri_result
+
+    - name: assert API works success
+      assert:
+        that:
+           - 'uri_result'
+
+    - name: check nonexistent endpoints cause errors
+      uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/nominal"
+      register: bad_uri_result
+      ignore_errors: true
+
+    - name: assert 
+      assert:
+        that:
+          - bad_uri_result|failed
+
+    # ============================================================
+
+    - name: deploy first API
+      aws_api_gateway:
+        api_file: "{{output_dir}}/minimal-swagger-api.yml"
+        stage: "minimal"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result_1
+
+    - name: deploy second API rapidly after first
+      aws_api_gateway:
+        api_file: "{{output_dir}}/minimal-swagger-api.yml"
+        stage: "minimal"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result_2
+
+    - name: assert both APIs deployed successfully
+      assert:
+        that:
+           - 'create_result_1.changed == True'
+           - 'create_result_2.changed == True'
+           - '"api_id" in create_result_1'
+           - '"api_id" in create_result_1'
+#           - '"created_response.created_date" in create_result'
+#           - '"deploy_response.created_date" in create_result'
+
+    - name: destroy first API
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result_1.api_id}}'
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: destroy_result_1
+
+    - name: destroy second API rapidly after first
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result_2.api_id}}'
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: destroy_result_2
+
+    - name: assert both APIs deployed successfully
+      assert:
+        that:
+           - 'destroy_result_1.changed == True'
+           - 'destroy_result_2.changed == True'
+#           - '"created_response.created_date" in create_result'
+#           - '"deploy_response.created_date" in create_result'
+
+  always:
+
+    # ============================================================
+    - name: test state=absent (expect changed=false)
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result.api_id}}'        
+        ec2_region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: destroy_result

test/integration/targets/aws_api_gateway/templates/minimal-swagger-api.yml.j2

@@ -0,0 +1,33 @@
+---
+swagger: "2.0"
+info:
+  version: "2017-05-11T12:14:59Z"
+  title: "{{resource_prefix}}Empty_API"
+host: "fakeexample.execute-api.us-east-1.amazonaws.com"
+basePath: "/minimal"
+schemes:
+- "https"
+paths:
+  /:
+    get:
+      consumes:
+      - "application/json"
+      produces:
+      - "application/json"
+      responses:
+        200:
+          description: "200 response"
+          schema:
+            $ref: "#/definitions/Empty"
+      x-amazon-apigateway-integration:
+        responses:
+          default:
+            statusCode: "200"
+        requestTemplates:
+          application/json: "{\"statusCode\": 200}"
+        passthroughBehavior: "when_no_match"
+        type: "mock"
+definitions:
+  Empty:
+    type: "object"
+    title: "Empty Schema"

test/units/modules/cloud/amazon/test_api_gateway.py

@@ -0,0 +1,85 @@
+#
+# (c) 2016 Michael De La Rue
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+
+from __future__ import (absolute_import, division, print_function)
+
+from nose.plugins.skip import SkipTest
+import pytest
+import sys
+import json
+from ansible.module_utils._text import to_bytes
+from ansible.module_utils import basic
+from ansible.module_utils.ec2 import HAS_BOTO3
+
+if not HAS_BOTO3:
+    raise SkipTest("test_api_gateway.py requires the `boto3` and `botocore` modules")
+
+import ansible.modules.cloud.amazon.aws_api_gateway as agw
+
+
+def set_module_args(args):
+    args = json.dumps({'ANSIBLE_MODULE_ARGS': args})
+    basic._ANSIBLE_ARGS = to_bytes(args)
+
+
+exit_return_dict = {}
+
+
+def fake_exit_json(self, **kwargs):
+    """ store the kwargs given to exit_json rather than putting them out to stdout"""
+    global exit_return_dict
+    exit_return_dict = kwargs
+    sys.exit(0)
+
+
+def test_upload_api(monkeypatch):
+    class FakeConnection:
+
+        def put_rest_api(self, *args, **kwargs):
+            assert kwargs["body"] == "the-swagger-text-is-fake"
+            return {"msg": "success!"}
+
+    def return_fake_connection(*args, **kwargs):
+        return FakeConnection()
+
+    monkeypatch.setattr(agw, "boto3_conn", return_fake_connection)
+    monkeypatch.setattr(agw.AnsibleModule, "exit_json", fake_exit_json)
+
+    set_module_args({
+        "api_id": "fred",
+        "state": "present",
+        "swagger_text": "the-swagger-text-is-fake",
+        "region": 'mars-north-1',
+    })
+    with pytest.raises(SystemExit):
+        agw.main()
+    assert exit_return_dict["changed"]
+
+
+def test_warn_if_region_not_specified():
+
+    set_module_args({
+        "name": "aws_api_gateway",
+        "state": "present",
+        "runtime": 'python2.7',
+        "role": 'arn:aws:iam::987654321012:role/lambda_basic_execution',
+        "handler": 'lambda_python.my_handler'})
+    with pytest.raises(SystemExit):
+        print(agw.main())