efs_facts: improve performance by reducing the number of api calls (#36520)

* efs_facts: improve performance by reducing the number of api calls

* Remove efs_facts tests from running in CI

hacking/aws_config/testing_policies/efs-policy.json

@@ -0,0 +1,36 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "ManageNetwork",
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateNetworkInterface",
+                "ec2:CreateSubnet",
+                "ec2:CreateTags",
+                "ec2:CreateVpc",
+                "ec2:DeleteNetworkInterface",
+                "ec2:DeleteSubnet",
+                "ec2:DeleteVpc",
+                "ec2:DescribeNetworkInterfaceAttribute",
+                "ec2:DescribeNetworkInterfaces",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeTags",
+                "ec2:DescribeVpcAttribute",
+                "ec2:DescribeVpcClassicLink",
+                "ec2:DescribeVpcs",
+                "ec2:ModifyVpcAttribute"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Sid": "ManageEFS",
+            "Effect": "Allow",
+            "Action": [
+                "elasticfilesystem:*"
+            ],
+            "Resource": "*"
+        }
+    ]
+}

lib/ansible/modules/cloud/amazon/efs_facts.py

@@ -209,6 +209,33 @@ class EFSConnection(object):
         """
         return self.connection.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups']
 
+    def get_mount_targets_data(self, file_systems):
+        for item in file_systems:
+            if item['life_cycle_state'] == self.STATE_AVAILABLE:
+                try:
+                    mount_targets = self.get_mount_targets(item['file_system_id'])
+                except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                    self.module.fail_json_aws(e, msg="Couldn't get EFS targets")
+                for mt in mount_targets:
+                    item['mount_targets'].append(camel_dict_to_snake_dict(mt))
+        return file_systems
+
+    def get_security_groups_data(self, file_systems):
+        for item in file_systems:
+            if item['life_cycle_state'] == self.STATE_AVAILABLE:
+                for target in item['mount_targets']:
+                    if target['life_cycle_state'] == self.STATE_AVAILABLE:
+                        try:
+                            target['security_groups'] = self.get_security_groups(target['mount_target_id'])
+                        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                            self.module.fail_json_aws(e, msg="Couldn't get EFS security groups")
+                    else:
+                        target['security_groups'] = []
+            else:
+                item['tags'] = {}
+                item['mount_targets'] = []
+        return file_systems
+
     def get_file_systems(self, file_system_id=None, creation_token=None):
         kwargs = dict()
         if file_system_id:
@@ -230,23 +257,9 @@ class EFSConnection(object):
             item['MountPoint'] = '.%s.efs.%s.amazonaws.com:/' % (item['FileSystemId'], self.region)
             if 'Timestamp' in item['SizeInBytes']:
                 item['SizeInBytes']['Timestamp'] = str(item['SizeInBytes']['Timestamp'])
-            if item['LifeCycleState'] == self.STATE_AVAILABLE:
-                try:
-                    item['MountTargets'] = self.get_mount_targets(item['FileSystemId'])
-                except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-                    self.module.fail_json_aws(e, msg="Couldn't get EFS targets")
-                for target in item['MountTargets']:
-                    if target['LifeCycleState'] == self.STATE_AVAILABLE:
-                        try:
-                            target['SecurityGroups'] = self.get_security_groups(target['MountTargetId'])
-                        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-                            self.module.fail_json_aws(e, msg="Couldn't get EFS security groups")
-                    else:
-                        target['SecurityGroups'] = []
-            else:
-                item['tags'] = {}
-                item['mount_targets'] = []
             result = camel_dict_to_snake_dict(item)
+            result['tags'] = {}
+            result['mount_targets'] = []
             # Set tags *after* doing camel to snake
             if result['life_cycle_state'] == self.STATE_AVAILABLE:
                 try:
@@ -340,6 +353,9 @@ def main():
     if tags:
         file_systems_info = [item for item in file_systems_info if has_tags(item['tags'], tags)]
 
+    file_systems_info = connection.get_mount_targets_data(file_systems_info)
+    file_systems_info = connection.get_security_groups_data(file_systems_info)
+
     if targets:
         targets = [(item, prefix_to_attr(item)) for item in targets]
         file_systems_info = [item for item in file_systems_info if has_targets(item['mount_targets'], targets)]

test/integration/targets/efs_facts/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+unsupported

test/integration/targets/efs_facts/tasks/main.yml

@@ -0,0 +1,241 @@
+---
+- block:
+
+    # ============================================================
+    - name: set connection information for all tasks
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: true
+
+    - name: Create VPC for testing
+      ec2_vpc_net:
+        name: "{{ resource_prefix }}-vpc"
+        cidr_block: 10.22.32.0/23
+        tags:
+          Name: Ansible ec2_instance Testing VPC
+        tenancy: default
+        <<: *aws_connection_info
+      register: testing_vpc
+
+    - name: Create subnet in zone A for testing
+      ec2_vpc_subnet:
+        state: present
+        vpc_id: "{{ testing_vpc.vpc.id }}"
+        cidr: 10.22.32.0/24
+        az: "{{ aws_region }}a"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet-a"
+        <<: *aws_connection_info
+      register: testing_subnet_a
+
+    - name: Create subnet in zone B for testing
+      ec2_vpc_subnet:
+        state: present
+        vpc_id: "{{ testing_vpc.vpc.id }}"
+        cidr: 10.22.33.0/24
+        az: "{{ aws_region }}b"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet-b"
+        <<: *aws_connection_info
+      register: testing_subnet_b
+
+    - name: Get default security group id for vpc
+      ec2_group_facts:
+        <<: *aws_connection_info
+        filters:
+          vpc-id: "{{ testing_vpc.vpc.id }}"
+      register: sg_facts
+
+    - set_fact:
+        vpc_default_sg_id: "{{sg_facts.security_groups[0].group_id}}"
+
+
+    # ============================================================
+    - name: Create Efs for testing
+      efs:
+        <<: *aws_connection_info
+        state: present
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+            Name: "{{ resource_prefix }}-test-tag"
+            Purpose: file-storage
+        targets:
+            - subnet_id: "{{testing_subnet_a.subnet.id}}"
+            - subnet_id: "{{testing_subnet_b.subnet.id}}"
+      register: created_efs
+
+    # ============================================================
+    - name: Get all EFS Facts
+      efs_facts:
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that:
+          - (efs_result.ansible_facts.efs | length) >= 1
+
+    # ============================================================
+    - name: Get EFS by creation token
+      efs_facts:
+        name: "{{ resource_prefix }}-test-efs"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - set_fact:
+        efs_result_assertions:
+          - efs_result is not changed
+          - (efs_result.ansible_facts.efs | length) == 1
+          - efs_result.ansible_facts.efs[0].creation_token == "{{ resource_prefix }}-test-efs"
+          - efs_result.ansible_facts.efs[0].file_system_id == created_efs.efs.file_system_id
+          - efs_result.ansible_facts.efs[0].number_of_mount_targets == 2
+          - (efs_result.ansible_facts.efs[0].mount_targets | length) == 2
+          - efs_result.ansible_facts.efs[0].name == "{{ resource_prefix }}-test-tag"
+          - efs_result.ansible_facts.efs[0].tags.Name == "{{ resource_prefix }}-test-tag"
+          - efs_result.ansible_facts.efs[0].tags.Purpose == "file-storage"
+          - efs_result.ansible_facts.efs[0].encrypted == false
+          - efs_result.ansible_facts.efs[0].life_cycle_state == "available"
+          - efs_result.ansible_facts.efs[0].performance_mode == "generalPurpose"
+          - efs_result.ansible_facts.efs[0].mount_targets[0].security_groups[0] == vpc_default_sg_id
+          - efs_result.ansible_facts.efs[0].mount_targets[1].security_groups[0] == vpc_default_sg_id
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
+    # ============================================================
+    - name: Get EFS by id
+      efs_facts:
+        id: "{{created_efs.efs.file_system_id}}"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
+    # ============================================================
+    - name: Get EFS by tag
+      efs_facts:
+        tags:
+          Name: "{{ resource_prefix }}-test-tag"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
+    # ============================================================
+    - name: Get EFS by target (subnet_id)
+      efs_facts:
+        targets:
+          - "{{testing_subnet_a.subnet.id}}"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
+    # ============================================================
+    - name: Get EFS by target (security_group_id)
+      efs_facts:
+        targets:
+          - "{{vpc_default_sg_id}}"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
+    # ============================================================
+    - name: Get EFS by tag and target
+      efs_facts:
+        tags:
+          Name: "{{ resource_prefix }}-test-tag"
+        targets:
+          - "{{testing_subnet_a.subnet.id}}"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
+    # ============================================================
+    - name: Query unknown EFS by tag
+      efs_facts:
+        tags:
+          Name: "{{ resource_prefix }}-unknown"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that:
+          - efs_result is not changed
+          - (efs_result.ansible_facts.efs | length) == 0
+
+    - name: Query unknown EFS by target
+      efs_facts:
+        targets:
+          - sg-00000000000
+        <<: *aws_connection_info
+      register: efs_result
+
+    - assert:
+        that:
+          - efs_result is not changed
+          - (efs_result.ansible_facts.efs | length) == 0
+
+  # ============================================================
+  always:
+    - name: Delete EFS used for tests
+      efs:
+        <<: *aws_connection_info
+        state: absent
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+          Name: "{{ resource_prefix }}-test-tag"
+          Purpose: file-storage
+      register: removed
+      until: removed is not failed
+      ignore_errors: yes
+      retries: 10
+
+    - name: Remove test subnet in zone A
+      ec2_vpc_subnet:
+        state: absent
+        vpc_id: "{{ testing_vpc.vpc.id }}"
+        cidr: 10.22.32.0/24
+        az: "{{ aws_region }}a"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet-a"
+        <<: *aws_connection_info
+      register: removed
+      until: removed is not failed
+      ignore_errors: yes
+      retries: 10
+
+    - name: Remove test subnet in zone B
+      ec2_vpc_subnet:
+        state: absent
+        vpc_id: "{{ testing_vpc.vpc.id }}"
+        cidr: 10.22.33.0/24
+        az: "{{ aws_region }}b"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet-b"
+        <<: *aws_connection_info
+      register: removed
+      until: removed is not failed
+      ignore_errors: yes
+      retries: 10
+
+    - name: remove the VPC
+      ec2_vpc_net:
+        name: "{{ resource_prefix }}-vpc"
+        cidr_block: 10.22.32.0/23
+        state: absent
+        <<: *aws_connection_info
+      register: removed
+      until: removed is not failed
+      ignore_errors: yes
+      retries: 10