[cloud] Update ec2_vpc_net_facts to use Boto3 (#25375)

* update ec2_vpc_net_facts module to boto3

updated with RETURN values and other requested changes

removed errant extra blank line

another errant extra line removed...auto-linter not working apparently

updates per review

fix typo in RETURN docs

* fix trailing whitespace issue

lib/ansible/modules/cloud/amazon/ec2_vpc_net_facts.py

@@ -26,14 +26,18 @@ description:
     - Gather facts about ec2 VPCs in AWS
 version_added: "2.1"
 author: "Rob White (@wimnat)"
+requirements:
+  - boto3
+  - botocore
 options:
+  vpc_ids:
+    description:
+      - A list of VPC IDs that exist in your account.
+    version_added: "2.5"
   filters:
     description:
       - A dict of filters to apply. Each dict item consists of a filter key and a filter value.
         See U(http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html) for possible filters.
-    required: false
-    default: null
-
 extends_documentation_fragment:
     - aws
     - ec2
@@ -47,8 +51,7 @@ EXAMPLES = '''
 
 # Gather facts about a particular VPC using VPC ID
 - ec2_vpc_net_facts:
-    filters:
+    vpc_ids: vpc-00112233
-      vpc-id: vpc-00112233
 
 # Gather facts about any VPC with a tag key Name and value Example
 - ec2_vpc_net_facts:
@@ -56,79 +59,207 @@ EXAMPLES = '''
       "tag:Name": Example
 
 '''
+
+RETURN = '''
+vpcs:
+    description: Returns an array of complex objects as described below.
+    returned: success
+    type: complex
+    contains:
+        id:
+            description: The ID of the VPC (for backwards compatibility).
+            returned: always
+            type: string
+        vpc_id:
+            description: The ID of the VPC .
+            returned: always
+            type: string
+        state:
+            description: The state of the VPC.
+            returned: always
+            type: string
+        tags:
+            description: A dict of tags associated with the VPC.
+            returned: always
+            type: dict
+        instance_tenancy:
+            description: The instance tenancy setting for the VPC.
+            returned: always
+            type: string
+        is_default:
+            description: True if this is the default VPC for account.
+            returned: always
+            type: boolean
+        cidr_block:
+            description: The IPv4 CIDR block assigned to the VPC.
+            returned: always
+            type: string
+        classic_link_dns_supported:
+            description: True/False depending on attribute setting for classic link DNS support.
+            returned: always
+            type: boolean
+        classic_link_enabled:
+            description: True/False depending on if classic link support is enabled.
+            returned: always
+            type: boolean
+        enable_dns_hostnames:
+            description: True/False depending on attribute setting for DNS hostnames support.
+            returned: always
+            type: boolean
+        enable_dns_support:
+            description: True/False depending on attribute setting for DNS support.
+            returned: always
+            type: boolean
+        ipv6_cidr_block_association_set:
+            description: An array of IPv6 cidr block association set information.
+            returned: always
+            type: complex
+            contains:
+                association_id:
+                    description: The association ID
+                    returned: always
+                    type: string
+                ipv6_cidr_block:
+                    description: The IPv6 CIDR block that is associated with the VPC.
+                    returned: always
+                    type: string
+                ipv6_cidr_block_state:
+                    description: A hash/dict that contains a single item. The state of the cidr block association.
+                    returned: always
+                    type: dict
+                    contains:
+                        state:
+                            description: The CIDR block association state.
+                            returned: always
+                            type: string
+'''
+
 import traceback
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.ec2 import (
+    boto3_conn,
+    ec2_argument_spec,
+    get_aws_connection_info,
+    AWSRetry,
+    HAS_BOTO3,
+    boto3_tag_list_to_ansible_dict,
+    camel_dict_to_snake_dict,
+    ansible_dict_to_boto3_filter_list
+)
 
 try:
-    import boto.vpc
+    import botocore
-    from boto.exception import BotoServerError
-    HAS_BOTO = True
 except ImportError:
-    HAS_BOTO = False
+    pass  # caught by imported HAS_BOTO3
-
-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.ec2 import connect_to_aws, ec2_argument_spec, get_aws_connection_info
-from ansible.module_utils._text import to_native
 
 
-def get_vpc_info(vpc):
+@AWSRetry.exponential_backoff()
+def describe_vpc_attr_with_backoff(connection, vpc_id, vpc_attribute):
+    """
+    Describe VPC Attributes with AWSRetry backoff throttling support.
+
+    connection  : boto3 client connection object
+    vpc_id      : The VPC ID to pull attribute value from
+    vpc_attribute     : The VPC attribute to get the value from - valid options = enableDnsSupport or enableDnsHostnames
+    """
+
+    return connection.describe_vpc_attribute(VpcId=vpc_id, Attribute=vpc_attribute)
+
+
+def describe_vpcs(connection, module):
+    """
+    Describe VPCs.
+
+    connection  : boto3 client connection object
+    module  : AnsibleModule object
+    """
+    # collect parameters
+    filters = ansible_dict_to_boto3_filter_list(module.params.get('filters'))
+    vpc_ids = module.params.get('vpc_ids')
+
+    # init empty list for return vars
+    vpc_info = list()
+    vpc_list = list()
+
+    # Get the basic VPC info
+    try:
+        response = connection.describe_vpcs(VpcIds=vpc_ids, Filters=filters)
+    except botocore.exceptions.ClientError as e:
+        module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+
+    # Loop through results and create a list of VPC IDs
+    for vpc in response['Vpcs']:
+        vpc_list.append(vpc['VpcId'])
+
+    # We can get these results in bulk but still needs two separate calls to the API
+    try:
+        cl_enabled = connection.describe_vpc_classic_link(VpcIds=vpc_list)
+    except botocore.exceptions.ClientError as e:
+        module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
 
     try:
-        classic_link = vpc.classic_link_enabled
+        cl_dns_support = connection.describe_vpc_classic_link_dns_support(VpcIds=vpc_list)
-    except AttributeError:
+    except botocore.exceptions.ClientError as e:
-        classic_link = False
+        module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
 
-    vpc_info = { 'id': vpc.id,
+    # Loop through the results and add the other VPC attributes we gathered
-                 'instance_tenancy': vpc.instance_tenancy,
+    for vpc in response['Vpcs']:
-                 'classic_link_enabled': classic_link,
+        # We have to make two separate calls per VPC to get these attributes.
-                 'dhcp_options_id': vpc.dhcp_options_id,
+        try:
-                 'state': vpc.state,
+            dns_support = describe_vpc_attr_with_backoff(connection, vpc['VpcId'], 'enableDnsSupport')
-                 'is_default': vpc.is_default,
+        except botocore.exceptions.ClientError as e:
-                 'cidr_block': vpc.cidr_block,
+            module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
-                 'tags': vpc.tags
-               }
 
-    return vpc_info
+        try:
+            dns_hostnames = describe_vpc_attr_with_backoff(connection, vpc['VpcId'], 'enableDnsHostnames')
+        except botocore.exceptions.ClientError as e:
+            module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
 
-def list_ec2_vpcs(connection, module):
+        # loop through the ClassicLink Enabled results and add the value for the correct VPC
+        for item in cl_enabled['Vpcs']:
+            if vpc['VpcId'] == item['VpcId']:
+                vpc['ClassicLinkEnabled'] = item['ClassicLinkEnabled']
 
-    filters = module.params.get("filters")
+        # loop through the ClassicLink DNS support results and add the value for the correct VPC
-    vpc_dict_array = []
+        for item in cl_dns_support['Vpcs']:
+            if vpc['VpcId'] == item['VpcId']:
+                vpc['ClassicLinkDnsSupported'] = item['ClassicLinkDnsSupported']
 
-    try:
+        # add the two DNS attributes
-        all_vpcs = connection.get_all_vpcs(filters=filters)
+        vpc['EnableDnsSupport'] = dns_support['EnableDnsSupport'].get('Value')
-    except BotoServerError as e:
+        vpc['EnableDnsHostnames'] = dns_hostnames['EnableDnsHostnames'].get('Value')
-        module.fail_json(msg=e.message)
+        # for backwards compatibility
+        vpc['id'] = vpc['VpcId']
+        vpc_info.append(camel_dict_to_snake_dict(vpc))
+        # convert tag list to ansible dict
+        vpc_info[-1]['tags'] = boto3_tag_list_to_ansible_dict(vpc.get('Tags', []))
 
-    for vpc in all_vpcs:
+    module.exit_json(vpcs=vpc_info)
-        vpc_dict_array.append(get_vpc_info(vpc))
-
-    module.exit_json(vpcs=vpc_dict_array)
 
 
 def main():
     argument_spec = ec2_argument_spec()
-    argument_spec.update(
+    argument_spec.update(dict(
-        dict(
+        vpc_ids=dict(type='list', default=[]),
-            filters = dict(default=None, type='dict')
+        filters=dict(type='dict', default={})
-        )
+    ))
-    )
 
-    module = AnsibleModule(argument_spec=argument_spec)
+    module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True)
 
-    if not HAS_BOTO:
+    if not HAS_BOTO3:
-        module.fail_json(msg='boto required for this module')
+        module.fail_json(msg='boto3 and botocore are required for this module')
 
-    region, ec2_url, aws_connect_params = get_aws_connection_info(module)
+    region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True)
 
     if region:
         try:
-            connection = connect_to_aws(boto.vpc, region, **aws_connect_params)
+            connection = boto3_conn(module, conn_type='client', resource='ec2', region=region, endpoint=ec2_url, **aws_connect_params)
-        except (boto.exception.NoAuthHandlerFound, Exception) as e:
+        except (botocore.exceptions.NoCredentialsError, botocore.exceptions.ProfileNotFound) as e:
-            module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+            module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
     else:
         module.fail_json(msg="region must be specified")
 
-    list_ec2_vpcs(connection, module)
+    describe_vpcs(connection, module)
 
 
 if __name__ == '__main__':

test/sanity/pep8/legacy-files.txt

@@ -29,7 +29,6 @@ lib/ansible/modules/cloud/amazon/ec2_vol_facts.py
 lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_option.py
 lib/ansible/modules/cloud/amazon/ec2_vpc_nacl.py
 lib/ansible/modules/cloud/amazon/ec2_vpc_net.py
-lib/ansible/modules/cloud/amazon/ec2_vpc_net_facts.py
 lib/ansible/modules/cloud/amazon/ec2_vpc_peer.py
 lib/ansible/modules/cloud/amazon/ec2_vpc_vgw.py
 lib/ansible/modules/cloud/amazon/ec2_vpc_vgw_facts.py