[cloud] add summary feature and refactor AWS cloudfront_facts module (#20791)

* initial commit of cloudfront_facts.py

* modification as per review from @georgepsarakis

* fixed shippable build error

* fixed shippable build error

* removed wildcard imports and replaced with specific imports from ansible.module_utils.ec2 as advised by @georgepsarakis

* renamed all instances of cloud_front_origin_access_identity to origin_access_identity as advised by @georgepsarakis

* show facts based on alias and distribution id for easy referencing as advised by @ryansb. have done for both distribution and distribution_config

* fixed incorrect logic for default list_distributions, fixed list_distributions_by_web_acl - wasn't passing web_acl_id, fixed list_invalidations keyword args missing DistributionId

* fixed last fact added clobbering all previous facts for an alias or an id

* removed list_ prefix from list keys

* removed unnecessary boto fields. made list_distributions and list_streaming_distributions dictionaries with id/alias as key. fixed list_invalidations.

* initial commit of cloudfront_facts summary

* checks for empty list returned from boto, standardised list naming

* neatened up parameters

* added summary documentation

* refactoring of functions for modularity

* refactoring, neatening code, fix for if cname not present, added try-catch blocks

* more refactoring, cleaning

* more cleaning, allowed streaming distributions to be found by domain name alias

* removed unnecessary line

* fixed tabs

* fixed indentation

* removed trailing whitespace

* removed more whitespace

* more refactoring, modified where invalidations dict is set

* added ETag to summary

* refactored summary list

* moved list out of for loop

* trailing white space

* reverted line lengthening as advised by @willthames

* modified exceptions to multi line as advised by @willthames

* reverted variable spacing to be more pythonic'

* reverted spacing in YAML as advised by @ryansb

* reverted line spacing for parameters for correct blame attribution as advised by @ryansb

* removed white space

* more white space

* reverted line spacings for parameters as advised by @ryansb

* removed spaces between parameters as advised by @ryansb

lib/ansible/modules/cloud/amazon/cloudfront_facts.py

@@ -115,6 +115,11 @@ options:
             - Get a list of streaming distributions.
         required: false
         default: false
+    summary:
+        description:
+            - Returns a summary of all distributions, streaming distributions and origin_access_identities. This is the default behaviour if no option is selected.
+        required: false
+        default: false
 
 extends_documentation_fragment:
     - aws
@@ -124,6 +129,10 @@ extends_documentation_fragment:
 EXAMPLES = '''
 # Note: These examples do not set authentication details, see the AWS Guide for details.
 
+# Get a summary of distributions
+- cloudfront_facts:
+    summary: true
+
 # Get information about a distribution
 - cloudfront_facts:
     distribution: true
@@ -152,6 +161,11 @@ EXAMPLES = '''
     origin_access_identity: true
     origin_access_identity_id: my-cloudfront-origin-access-identity-id
 
+# Get all information about lists not requiring parameters (ie. list_origin_access_identities, list_distributions, list_streaming_distributions)
+- cloudfront_facts:
+    origin_access_identity: true
+    origin_access_identity_id: my-cloudfront-origin-access-identity-id
+
 # Get all information about lists not requiring parameters (ie. list_origin_access_identities, list_distributions, list_streaming_distributions)
 - cloudfront_facts:
     all_lists: true
@@ -186,6 +200,10 @@ streaming_distribution_configuration:
     description: Describes the streaming configuration information for the distribution. Requires distribution_id or domain_name_alias to be specified.
     returned: only if streaming_distribution_configuration is true
     type: dict
+summary:
+    description: Gives a summary of distributions, streaming distributions and origin access identities.
+    returned: as default or if summary is true
+    type: dict
 '''
 
 try:
@@ -217,122 +235,225 @@ class CloudFrontServiceManager:
         except botocore.exceptions.NoRegionError:
             self.module.fail_json(msg="Region must be specified as a parameter, in AWS_DEFAULT_REGION environment variable or in boto configuration file")
         except Exception as e:
-            self.module.fail_json(msg="Can't establish connection - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Can't establish connection - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_distribution(self, distribution_id):
         try:
             func = partial(self.client.get_distribution,Id=distribution_id)
             return self.paginated_response(func)
         except Exception as e:
-            self.module.fail_json(msg="Error describing distribution - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing distribution - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_distribution_config(self, distribution_id):
         try:
             func = partial(self.client.get_distribution_config,Id=distribution_id)
             return self.paginated_response(func)
         except Exception as e:
-            self.module.fail_json(msg="Error describing distribution configuration - " + str(e), exception=traceback.format_exec(e))
+            self.module.fail_json(msg="Error describing distribution configuration - " + str(e),
+                                  exception=traceback.format_exec(e))
 
     def get_origin_access_identity(self, origin_access_identity_id):
         try:
             func = partial(self.client.get_cloud_front_origin_access_identity,Id=origin_access_identity_id)
             return self.paginated_response(func)
         except Exception as e:
-            self.module.fail_json(msg="Error describing origin access identity - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing origin access identity - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_origin_access_identity_config(self, origin_access_identity_id):
         try:
             func = partial(self.client.get_cloud_front_origin_access_identity_config,Id=origin_access_identity_id)
             return self.paginated_response(func)
         except Exception as e:
-            self.module.fail_json(msg="Error describing origin access identity configuration - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing origin access identity configuration - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_invalidation(self, distribution_id, invalidation_id):
         try:
             func = partial(self.client.get_invalidation,DistributionId=distribution_id,Id=invalidation_id)
             return self.paginated_response(func)
         except Exception as e:
-            self.module.fail_json(msg="Error describing invalidation - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing invalidation - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_streaming_distribution(self, distribution_id):
         try:
             func = partial(self.client.get_streaming_distribution,Id=distribution_id)
             return self.paginated_response(func)
         except Exception as e:
-            self.module.fail_json(msg="Error describing streaming distribution - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing streaming distribution - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_streaming_distribution_config(self, distribution_id):
         try:
             func = partial(self.client.get_streaming_distribution_config,Id=distribution_id)
             return self.paginated_response(func)
         except Exception as e:
-            self.module.fail_json(msg="Error describing streaming distribution - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing streaming distribution - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def list_origin_access_identities(self):
         try:
             func = partial(self.client.list_cloud_front_origin_access_identities)
-            return self.paginated_response(func, 'CloudFrontOriginAccessIdentityList')['Items']
+            origin_access_identity_list = self.paginated_response(func, 'CloudFrontOriginAccessIdentityList')
+            if origin_access_identity_list['Quantity'] > 0:
+              return origin_access_identity_list['Items']
+            return {}
         except Exception as e:
-            self.module.fail_json(msg="Error listing cloud front origin access identities = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing cloud front origin access identities - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def list_distributions(self, keyed=True):
         try:
             func = partial(self.client.list_distributions)
-            distribution_list = self.paginated_response(func, 'DistributionList')['Items']
+            distribution_list = self.paginated_response(func, 'DistributionList')
+            if distribution_list['Quantity'] == 0:
+                return {}
+            else:
+                distribution_list = distribution_list['Items']
             if not keyed:
                 return distribution_list
             return self.keyed_list_helper(distribution_list)
         except Exception as e:
-            self.module.fail_json(msg="Error listing distributions = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing distributions - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def list_distributions_by_web_acl_id(self, web_acl_id):
         try:
             func = partial(self.client.list_distributions_by_web_acl_id, WebAclId=web_acl_id)
-            distributions = self.paginated_response(func, 'DistributionList')['Items']
+            distribution_list = self.paginated_response(func, 'DistributionList')
-            return self.keyed_list_helper(distributions)
+            if distribution_list['Quantity'] == 0:
+                return {}
+            else:
+                distribution_list = distribution_list['Items']
+            return self.keyed_list_helper(distribution_list)
         except Exception as e:
-            self.module.fail_json(msg="Error listing distributions by web acl id = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing distributions by web acl id - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def list_invalidations(self, distribution_id):
         try:
             func = partial(self.client.list_invalidations, DistributionId=distribution_id)
-            return self.paginated_response(func, 'InvalidationList')['Items']
+            invalidation_list = self.paginated_response(func, 'InvalidationList')
+            if invalidation_list['Quantity'] > 0:
+                return invalidation_list['Items']
+            return {}
         except Exception as e:
-            self.module.fail_json(msg="Error listing invalidations = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing invalidations - " + str(e),
+                                  exception=traceback.format_exc(e))
 
-    def list_streaming_distributions(self):
+    def list_streaming_distributions(self, keyed=True):
         try:
             func = partial(self.client.list_streaming_distributions)
-            streaming_distributions = self.paginated_response(func, 'StreamingDistributionList')['Items']
+            streaming_distribution_list = self.paginated_response(func, 'StreamingDistributionList')
-            return self.keyed_list_helper(streaming_distributions)
+            if streaming_distribution_list['Quantity'] == 0:
+                return {}
+            else:
+                streaming_distribution_list = streaming_distribution_list['Items']
+            if not keyed:
+                return streaming_distribution_list
+            return self.keyed_list_helper(streaming_distribution_list)
         except Exception as e:
-            self.module.fail_json(msg="Error listing streaming distributions = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing streaming distributions - " + str(e),
+                                  exception=traceback.format_exc(e))
+
+    def summary(self):
+        summary_dict = {}
+        summary_dict.update(self.summary_get_distribution_list(False))
+        summary_dict.update(self.summary_get_distribution_list(True))
+        summary_dict.update(self.summary_get_origin_access_identity_list())
+        return summary_dict
+
+    def summary_get_origin_access_identity_list(self):
+        try:
+            origin_access_identity_list = { 'origin_access_identities': [] }
+            origin_access_identities = self.list_origin_access_identities()
+            for origin_access_identity in origin_access_identities:
+                oai_id = origin_access_identity['Id']
+                oai_full_response = self.get_origin_access_identity(oai_id)
+                oai_summary = { 'Id': oai_id, 'ETag': oai_full_response['ETag'] }
+                origin_access_identity_list['origin_access_identities'].append( oai_summary )
+            return origin_access_identity_list
+        except Exception as e:
+            self.module.fail_json(msg="Error generating summary of origin access identities - " + str(e),
+                                  exception=traceback.format_exc(e))
+
+    def summary_get_distribution_list(self, streaming=False):
+        try:
+            list_name = 'streaming_distributions' if streaming else 'distributions'
+            key_list = ['Id', 'ARN', 'Status', 'LastModifiedTime', 'DomainName', 'Comment', 'PriceClass', 'Enabled' ]
+            distribution_list = { list_name: [] }
+            distributions = self.list_streaming_distributions(False) if streaming else self.list_distributions(False)
+            for dist in distributions:
+                temp_distribution = {}
+                for key_name in key_list:
+                    temp_distribution.update( { key_name: dist[key_name] } )
+                temp_distribution.update( { 'Aliases': [] } )
+                temp_distribution.update( { 'ETag': self.get_etag_from_distribution_id(dist['Id'], streaming) } )
+                if 'Items' in dist['Aliases']:
+                    for alias in dist['Aliases']['Items']:
+                        temp_distribution['Aliases'].append(alias)
+                if not streaming:
+                    temp_distribution.update( { 'WebACLId': dist['WebACLId'] } )
+                    invalidation_ids = self.get_list_of_invalidation_ids_from_distribution_id(dist['Id'])
+                    if invalidation_ids:
+                        temp_distribution.update( { 'Invalidations': invalidation_ids } )
+                distribution_list[list_name].append(temp_distribution)
+            return distribution_list
+        except Exception as e:
+            self.module.fail_json(msg="Error generating summary of distributions - " + str(e),
+                                  exception=traceback.format_exc(e))
+
+    def get_etag_from_distribution_id(self, distribution_id, streaming):
+        distribution = {}
+        if not streaming:
+            distribution = self.get_distribution(distribution_id)
+        else:
+            distribution = self.get_streaming_distribution(distribution_id)
+        return distribution['ETag']
+
+    def get_list_of_invalidation_ids_from_distribution_id(self, distribution_id):
+        try:
+            invalidation_ids = []
+            invalidations = self.list_invalidations(distribution_id)
+            for invalidation in invalidations:
+                invalidation_ids.append(invalidation['Id'])
+            return invalidation_ids
+        except Exception as e:
+            self.module.fail_json(msg="Error getting list of invalidation ids - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_distribution_id_from_domain_name(self, domain_name):
         try:
             distribution_id = ""
             distributions = self.list_distributions(False)
+            distributions += self.list_streaming_distributions(False)
             for dist in distributions:
+                if 'Items' in dist['Aliases']:
                     for alias in dist['Aliases']['Items']:
                         if str(alias).lower() == domain_name.lower():
-                        distribution_id = str(dist['Id'])
+                            distribution_id = dist['Id']
                             break
             return distribution_id
         except Exception as e:
-            self.module.fail_json(msg="Error getting distribution id from domain name = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error getting distribution id from domain name - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def get_aliases_from_distribution_id(self, distribution_id):
         aliases = []
         try:
             distributions = self.list_distributions(False)
             for dist in distributions:
-                if dist['Id'] == distribution_id:
+                if dist['Id'] == distribution_id and 'Items' in dist['Aliases']:
                     for alias in dist['Aliases']['Items']:
                         aliases.append(alias)
                     break
             return aliases
         except Exception as e:
-            self.module.fail_json(msg="Error getting list of aliases from distribution_id = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error getting list of aliases from distribution_id - " + str(e),
+                                  exception=traceback.format_exc(e))
 
     def paginated_response(self, func, result_key=""):
         '''
@@ -357,13 +478,20 @@ class CloudFrontServiceManager:
     def keyed_list_helper(self, list_to_key):
         keyed_list = dict()
         for item in list_to_key:
-            aliases = item['Aliases']['Items']
             distribution_id = item['Id']
-            keyed_list.update({distribution_id: item})
+            if 'Items' in item['Aliases']:
+                aliases = item['Aliases']['Items']
                 for alias in aliases:
                     keyed_list.update( { alias: item } )
+            keyed_list.update( { distribution_id: item } )
         return keyed_list
 
+def set_facts_for_distribution_id_and_alias(details, facts, distribution_id, aliases):
+    facts[distribution_id].update(details)
+    for alias in aliases:
+        facts[alias].update(details)
+    return facts
+
 def main():
     argument_spec = ec2_argument_spec()
     argument_spec.update(dict(
@@ -383,7 +511,8 @@ def main():
         list_distributions=dict(required=False, default=False, type='bool'),
         list_distributions_by_web_acl_id=dict(required=False, default=False, type='bool'),
         list_invalidations=dict(required=False, default=False, type='bool'),
-        list_streaming_distributions=dict(required=False, default=False, type='bool')
+        list_streaming_distributions=dict(required=False, default=False, type='bool'),
+        summary=dict(required=False, default=False, type='bool')
     ))
 
     module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False)
@@ -398,9 +527,7 @@ def main():
     origin_access_identity_id = module.params.get('origin_access_identity_id')
     web_acl_id = module.params.get('web_acl_id')
     domain_name_alias = module.params.get('domain_name_alias')
-
     all_lists = module.params.get('all_lists')
-
     distribution = module.params.get('distribution')
     distribution_config = module.params.get('distribution_config')
     origin_access_identity = module.params.get('origin_access_identity')
@@ -408,23 +535,25 @@ def main():
     invalidation = module.params.get('invalidation')
     streaming_distribution = module.params.get('streaming_distribution')
     streaming_distribution_config = module.params.get('streaming_distribution_config')
-
     list_origin_access_identities = module.params.get('list_origin_access_identities')
     list_distributions = module.params.get('list_distributions')
     list_distributions_by_web_acl_id = module.params.get('list_distributions_by_web_acl_id')
     list_invalidations = module.params.get('list_invalidations')
     list_streaming_distributions = module.params.get('list_streaming_distributions')
+    summary = module.params.get('summary')
 
     aliases = []
+    result = { 'cloudfront': {} }
+    facts = {}
 
-    require_distribution_id = (distribution or distribution_config or invalidation or
+    require_distribution_id = (distribution or distribution_config or invalidation or streaming_distribution or
-            streaming_distribution or streaming_distribution_config or list_invalidations)
+        streaming_distribution_config or list_invalidations)
 
-    # set default to list_distributions if no option specified
+    # set default to summary if no option specified
-    list_distributions = list_distributions or not (distribution or distribution_config or
+    summary = summary or not (distribution or distribution_config or origin_access_identity or
-            origin_access_identity or origin_access_identity_config or invalidation or
+        origin_access_identity_config or invalidation or streaming_distribution or streaming_distribution_config or
-            streaming_distribution or streaming_distribution_config or list_origin_access_identities or
+        list_origin_access_identities or list_distributions_by_web_acl_id or list_invalidations or
-            list_distributions_by_web_acl_id or list_invalidations or list_streaming_distributions)
+        list_streaming_distributions or list_distributions)
 
     # validations
     if require_distribution_id and distribution_id is None and domain_name_alias is None:
@@ -444,65 +573,42 @@ def main():
 
     # set appropriate cloudfront id
     if distribution_id and not list_invalidations:
-        result = { 'cloudfront': { distribution_id: {} } }
+        facts = { distribution_id: {} }
         aliases = service_mgr.get_aliases_from_distribution_id(distribution_id)
         for alias in aliases:
-            result['cloudfront'].update( { alias: {} } )
+            facts.update( { alias: {} } )
         if invalidation_id:
-            result['cloudfront'].update( { invalidation_id: {} } )
+            facts.update( { invalidation_id: {} } )
-        facts = result['cloudfront']
+    elif distribution_id and list_invalidations:
-    elif list_invalidations:
+        facts = { distribution_id: {} }
-        result = { 'cloudfront': { 'invalidations': {} } }
-        facts = result['cloudfront']['invalidations']
         aliases = service_mgr.get_aliases_from_distribution_id(distribution_id)
         for alias in aliases:
-            result['cloudfront']['invalidations'].update( { alias: {} } )
+            facts.update( { alias: {} } )
     elif origin_access_identity_id:
-        result = { 'cloudfront': { origin_access_identity_id: {} } }
+        facts = { origin_access_identity_id: {} }
-        facts = result['cloudfront'][origin_access_identity_id]
     elif web_acl_id:
-        result = { 'cloudfront': { web_acl_id: {} } }
+        facts = { web_acl_id: {} }
-        facts = result['cloudfront'][web_acl_id]
-    else:
-        result = { 'cloudfront': {} }
-        facts = result['cloudfront']
 
     # get details based on options
     if distribution:
-        distribution_details = service_mgr.get_distribution(distribution_id)
+        facts_to_set = service_mgr.get_distribution(distribution_id)
-        facts[distribution_id].update(distribution_details)
-        for alias in aliases:
-            facts[alias].update(distribution_details)
     if distribution_config:
-        distribution_config_details = service_mgr.get_distribution_config(distribution_id)
+        facts_to_set = service_mgr.get_distribution_config(distribution_id)
-        facts[distribution_id].update(distribution_config_details)
-        for alias in aliases:
-            facts[alias].update(distribution_config_details)
     if origin_access_identity:
         facts[origin_access_identity_id].update(service_mgr.get_origin_access_identity(origin_access_identity_id))
     if origin_access_identity_config:
         facts[origin_access_identity_id].update(service_mgr.get_origin_access_identity_config(origin_access_identity_id))
     if invalidation:
-        invalidation = service_mgr.get_invalidation(distribution_id, invalidation_id)
+        facts_to_set = service_mgr.get_invalidation(distribution_id, invalidation_id)
-        facts[invalidation_id].update(invalidation)
+        facts[invalidation_id].update(facts_to_set)
-        facts[distribution_id].update(invalidation)
-        for alias in aliases:
-            facts[alias].update(invalidation)
     if streaming_distribution:
-        streaming_distribution_details = service_mgr.get_streaming_distribution(distribution_id)
+        facts_to_set = service_mgr.get_streaming_distribution(distribution_id)
-        facts[distribution_id].update(streaming_distribution_details)
-        for alias in aliases:
-            facts[alias].update(streaming_distribution_details)
     if streaming_distribution_config:
-        streaming_distribution_config_details = service_mgr.get_streaming_distribution_config(distribution_id)
+        facts_to_set = service_mgr.get_streaming_distribution_config(distribution_id)
-        facts[distribution_id].update(streaming_distribution_config_details)
-        for alias in aliases:
-            facts[alias].update(streaming_distribution_config_details)
     if list_invalidations:
-        invalidations = service_mgr.list_invalidations(distribution_id)
+        facts_to_set = {'invalidations': service_mgr.list_invalidations(distribution_id) }
-        facts[distribution_id].update(invalidations)
+    if 'facts_to_set' in vars():
-        for alias in aliases:
+        facts = set_facts_for_distribution_id_and_alias(facts_to_set, facts, distribution_id, aliases)
-            facts[alias].update(invalidations)
 
     # get list based on options
     if all_lists or list_origin_access_identities:
@@ -513,8 +619,15 @@ def main():
         facts['streaming_distributions'] = service_mgr.list_streaming_distributions()
     if list_distributions_by_web_acl_id:
         facts['distributions_by_web_acl_id'] = service_mgr.list_distributions_by_web_acl_id(web_acl_id)
+    if list_invalidations:
+        facts['invalidations'] = service_mgr.list_invalidations(distribution_id)
+
+    # default summary option
+    if summary:
+        facts['summary'] = service_mgr.summary()
 
     result['changed'] = False
+    result['cloudfront'].update(facts)
     module.exit_json(msg="Retrieved cloudfront facts.", ansible_facts=result)
 
 if __name__ == '__main__':