New module: AWS Glue connection (#39492)

* New module = AWS Glue connection

* Add a few initial integration tests

* Add alias for CI

* module rename

* finish module rename

* add loop when getting glue connection again so we dont get None

* Limit number of retries to get new glue connection info

lib/ansible/modules/cloud/amazon/aws_glue_connection.py

@@ -0,0 +1,329 @@
+#!/usr/bin/python
+
+# Copyright: (c) 2018, Rob White (@wimnat)
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+DOCUMENTATION = '''
+---
+module: aws_glue_connection
+short_description: Manage an AWS Glue connection
+description:
+    - Manage an AWS Glue connection. See U(https://aws.amazon.com/glue/) for details.
+version_added: "2.6"
+requirements: [ boto3 ]
+author: "Rob White (@wimnat)"
+options:
+  catalog_id:
+    description:
+      - The ID of the Data Catalog in which to create the connection. If none is supplied,
+        the AWS account ID is used by default.
+    required: false
+  connection_properties:
+    description:
+      - A dict of key-value pairs used as parameters for this connection.
+    required: true
+  connection_type:
+    description:
+      - The type of the connection. Currently, only JDBC is supported; SFTP is not supported.
+    required: false
+    default: JDBC
+    choices: [ 'JDBC', 'SFTP' ]
+  description:
+    description:
+      - The description of the connection.
+    required: false
+  match_criteria:
+    description:
+      - A list of UTF-8 strings that specify the criteria that you can use in selecting this connection.
+    required: false
+  name:
+    description:
+      - The name of the connection.
+    required: true
+  security_groups:
+    description:
+      - A list of security groups to be used by the connection. Use either security group name or ID.
+    required: false
+  state:
+    description:
+      - Create or delete the AWS Glue connection.
+    required: true
+    choices: [ 'present', 'absent' ]
+  subnet_id:
+    description:
+      - The subnet ID used by the connection.
+    required: false
+extends_documentation_fragment:
+    - aws
+    - ec2
+'''
+
+EXAMPLES = '''
+# Note: These examples do not set authentication details, see the AWS Guide for details.
+
+# Create an AWS Glue connection
+- aws_glue_connection:
+    name: my-glue-connection
+    connection_properties:
+      JDBC_CONNECTION_URL: jdbc:mysql://mydb:3306/databasename
+      USERNAME: my-username
+      PASSWORD: my-password
+    state: present
+
+# Delete an AWS Glue connection
+- aws_glue_connection:
+    name: my-glue-connection
+    state: absent
+
+'''
+
+RETURN = '''
+connection_properties:
+    description: A dict of key-value pairs used as parameters for this connection.
+    returned: when state is present
+    type: dict
+    sample: {'JDBC_CONNECTION_URL':'jdbc:mysql://mydb:3306/databasename','USERNAME':'x','PASSWORD':'y'}
+connection_type:
+    description: The type of the connection.
+    returned: when state is present
+    type: string
+    sample: JDBC
+creation_time:
+    description: The time this connection definition was created.
+    returned: when state is present
+    type: string
+    sample: "2018-04-21T05:19:58.326000+00:00"
+description:
+    description: Description of the job being defined.
+    returned: when state is present
+    type: string
+    sample: My first Glue job
+last_updated_time:
+    description: The last time this connection definition was updated.
+    returned: when state is present
+    type: string
+    sample: "2018-04-21T05:19:58.326000+00:00"
+match_criteria:
+    description: A list of criteria that can be used in selecting this connection.
+    returned: when state is present
+    type: list
+    sample: []
+name:
+    description: The name of the connection definition.
+    returned: when state is present
+    type: string
+    sample: my-glue-connection
+physical_connection_requirements:
+    description: A dict of physical connection requirements, such as VPC and SecurityGroup,
+                 needed for making this connection successfully.
+    returned: when state is present
+    type: dict
+    sample: {'subnet-id':'subnet-aabbccddee'}
+'''
+
+from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.ec2 import camel_dict_to_snake_dict, get_ec2_security_group_ids_from_names
+
+# Non-ansible imports
+import copy
+import time
+try:
+    from botocore.exceptions import BotoCoreError, ClientError
+except ImportError:
+    pass
+
+
+def _get_glue_connection(connection, module):
+    """
+    Get an AWS Glue connection based on name. If not found, return None.
+
+    :param connection: AWS boto3 glue connection
+    :param module: Ansible module
+    :return: boto3 Glue connection dict or None if not found
+    """
+
+    connection_name = module.params.get("name")
+    connection_catalog_id = module.params.get("catalog_id")
+
+    params = {'Name': connection_name}
+    if connection_catalog_id is not None:
+        params['CatalogId'] = connection_catalog_id
+
+    try:
+        return connection.get_connection(**params)['Connection']
+    except (BotoCoreError, ClientError) as e:
+        if e.response['Error']['Code'] == 'EntityNotFoundException':
+            return None
+        else:
+            raise e
+
+
+def _compare_glue_connection_params(user_params, current_params):
+    """
+    Compare Glue connection params. If there is a difference, return True immediately else return False
+
+    :param user_params: the Glue connection parameters passed by the user
+    :param current_params: the Glue connection parameters currently configured
+    :return: True if any parameter is mismatched else False
+    """
+
+    # Weirdly, boto3 doesn't return some keys if the value is empty e.g. Description
+    # To counter this, add the key if it's missing with a blank value
+
+    if 'Description' not in current_params:
+        current_params['Description'] = ""
+    if 'MatchCriteria' not in current_params:
+        current_params['MatchCriteria'] = list()
+    if 'PhysicalConnectionRequirements' not in current_params:
+        current_params['PhysicalConnectionRequirements'] = dict()
+        current_params['PhysicalConnectionRequirements']['SecurityGroupIdList'] = []
+        current_params['PhysicalConnectionRequirements']['SubnetId'] = ""
+
+    if 'ConnectionProperties' in user_params['ConnectionInput'] and user_params['ConnectionInput']['ConnectionProperties'] \
+            != current_params['ConnectionProperties']:
+        return True
+    if 'ConnectionType' in user_params['ConnectionInput'] and user_params['ConnectionInput']['ConnectionType'] \
+            != current_params['ConnectionType']:
+        return True
+    if 'Description' in user_params['ConnectionInput'] and user_params['ConnectionInput']['Description'] != current_params['Description']:
+        return True
+    if 'MatchCriteria' in user_params['ConnectionInput'] and set(user_params['ConnectionInput']['MatchCriteria']) != set(current_params['MatchCriteria']):
+        return True
+    if 'PhysicalConnectionRequirements' in user_params['ConnectionInput']:
+        if 'SecurityGroupIdList' in user_params['ConnectionInput']['PhysicalConnectionRequirements'] and \
+                set(user_params['ConnectionInput']['PhysicalConnectionRequirements']['SecurityGroupIdList']) \
+                != set(current_params['PhysicalConnectionRequirements']['SecurityGroupIdList']):
+            return True
+        if 'SubnetId' in user_params['ConnectionInput']['PhysicalConnectionRequirements'] and \
+                user_params['ConnectionInput']['PhysicalConnectionRequirements']['SubnetId'] \
+                != current_params['PhysicalConnectionRequirements']['SubnetId']:
+            return True
+
+    return False
+
+
+def create_or_update_glue_connection(connection, connection_ec2, module, glue_connection):
+    """
+    Create or update an AWS Glue connection
+
+    :param connection: AWS boto3 glue connection
+    :param module: Ansible module
+    :param glue_connection: a dict of AWS Glue connection parameters or None
+    :return:
+    """
+
+    changed = False
+    params = dict()
+    params['ConnectionInput'] = dict()
+    params['ConnectionInput']['Name'] = module.params.get("name")
+    params['ConnectionInput']['ConnectionType'] = module.params.get("connection_type")
+    params['ConnectionInput']['ConnectionProperties'] = module.params.get("connection_properties")
+    if module.params.get("catalog_id") is not None:
+        params['CatalogId'] = module.params.get("catalog_id")
+    if module.params.get("description") is not None:
+        params['ConnectionInput']['Description'] = module.params.get("description")
+    if module.params.get("match_criteria") is not None:
+        params['ConnectionInput']['MatchCriteria'] = module.params.get("match_criteria")
+    if module.params.get("security_groups") is not None or module.params.get("subnet_id") is not None:
+        params['ConnectionInput']['PhysicalConnectionRequirements'] = dict()
+    if module.params.get("security_groups") is not None:
+        # Get security group IDs from names
+        security_group_ids = get_ec2_security_group_ids_from_names(module.params.get('security_groups'), connection_ec2, boto3=True)
+        params['ConnectionInput']['PhysicalConnectionRequirements']['SecurityGroupIdList'] = security_group_ids
+    if module.params.get("subnet_id") is not None:
+        params['ConnectionInput']['PhysicalConnectionRequirements']['SubnetId'] = module.params.get("subnet_id")
+
+    # If glue_connection is not None then check if it needs to be modified, else create it
+    if glue_connection:
+        if _compare_glue_connection_params(params, glue_connection):
+            try:
+                # We need to slightly modify the params for an update
+                update_params = copy.deepcopy(params)
+                update_params['Name'] = update_params['ConnectionInput']['Name']
+                connection.update_connection(**update_params)
+                changed = True
+            except (BotoCoreError, ClientError) as e:
+                module.fail_json_aws(e)
+    else:
+        try:
+            connection.create_connection(**params)
+            changed = True
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e)
+
+    # If changed, get the Glue connection again
+    if changed:
+        glue_connection = None
+        for i in range(10):
+            glue_connection = _get_glue_connection(connection, module)
+            if glue_connection is not None:
+                break
+            time.sleep(10)
+
+    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_connection))
+
+
+def delete_glue_connection(connection, module, glue_connection):
+    """
+    Delete an AWS Glue connection
+
+    :param connection: AWS boto3 glue connection
+    :param module: Ansible module
+    :param glue_connection: a dict of AWS Glue connection parameters or None
+    :return:
+    """
+
+    changed = False
+
+    params = {'ConnectionName': module.params.get("name")}
+    if module.params.get("catalog_id") is not None:
+        params['CatalogId'] = module.params.get("catalog_id")
+
+    if glue_connection:
+        try:
+            connection.delete_connection(**params)
+            changed = True
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e)
+
+    module.exit_json(changed=changed)
+
+
+def main():
+
+    argument_spec = (
+        dict(
+            catalog_id=dict(type='str'),
+            connection_properties=dict(type='dict'),
+            connection_type=dict(type='str', default='JDBC', choices=['JDBC', 'SFTP']),
+            description=dict(type='str'),
+            match_criteria=dict(type='list'),
+            name=dict(required=True, type='str'),
+            security_groups=dict(type='list'),
+            state=dict(required=True, choices=['present', 'absent'], type='str'),
+            subnet_id=dict(type='str')
+        )
+    )
+
+    module = AnsibleAWSModule(argument_spec=argument_spec,
+                              required_if=[
+                                  ('state', 'present', ['connection_properties'])
+                              ]
+                              )
+
+    connection_glue = module.client('glue')
+    connection_ec2 = module.client('ec2')
+
+    glue_connection = _get_glue_connection(connection_glue, module)
+
+    if module.params.get("state") == 'present':
+        create_or_update_glue_connection(connection_glue, connection_ec2, module, glue_connection)
+    else:
+        delete_glue_connection(connection_glue, module, glue_connection)
+
+if __name__ == '__main__':
+    main()

test/integration/targets/aws_glue_connection/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+posix/ci/cloud/group4/aws

test/integration/targets/aws_glue_connection/tasks/main.yml

@@ -0,0 +1,87 @@
+- block:
+
+    # TODO: description, match_criteria, security_groups, and subnet_id are unused module options
+
+    - name: set up aws connection info
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: yes
+
+    - name: create glue connection
+      aws_glue_connection:
+        name: "{{ resource_prefix }}"
+        connection_properties:
+          JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}"
+          USERNAME: my-username
+          PASSWORD: my-password
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - assert:
+        that:
+          - result.changed
+
+    - name: test idempotence creating glue connection
+      aws_glue_connection:
+        name: "{{ resource_prefix }}"
+        connection_properties:
+          JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}"
+          USERNAME: my-username
+          PASSWORD: my-password
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - assert:
+        that:
+          - not result.changed
+
+    - name: test updating JDBC connection url
+      aws_glue_connection:
+        name: "{{ resource_prefix }}"
+        connection_properties:
+          JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}-updated"
+          USERNAME: my-username
+          PASSWORD: my-password
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - assert:
+        that:
+          - result.changed
+
+    - name: delete glue connection
+      aws_glue_connection:
+        name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+
+    - assert:
+        that:
+          - result.changed
+
+    - name: test idempotence removing glue connection
+      aws_glue_connection:
+        name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+
+    - assert:
+        that:
+          - not result.changed
+
+  always:
+
+    - name: delete glue connection
+      aws_glue_connection:
+        name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info