EFS - add support for new Provisioned Throughput (#43253)

* efs.py: Add support for EFS provisioned throughput

* efs_facts.py: Add support for EFS provisioned throughput

* efs_facts integration tests updated with provision throughput

* efs_facts: Tests refactoring - add failure and success playbook according to botocore version.

* efs_facts: More tests and new option descriptions adjustment

* efs_facts tests renamed to efs

lib/ansible/modules/cloud/amazon/efs.py

@@ -70,6 +70,18 @@ options:
                    - ip_address - Optional. A valid IPv4 address within the address range of the specified subnet.
                    - security_groups - Optional. List of security group IDs, of the form 'sg-xxxxxxxx'. These must be for the same VPC as subnet specified
                This data may be modified for existing EFS using state 'present' and new list of mount targets."
+    throughput_mode:
+        description:
+            - The throughput_mode for the file system to be created.
+            - Requires botocore >= 1.10.57
+        choices: ['bursting', 'provisioned']
+        version_added: 2.8
+    provisioned_throughput_in_mibps:
+        description:
+            - If the throughput_mode is provisioned, select the amount of throughput to provisioned in Mibps.
+            - Requires botocore >= 1.10.57
+        type: float
+        version_added: 2.8
     wait:
         description:
             - "In case of 'present' state should wait for EFS 'available' life cycle state (of course, if current state not 'deleting' or 'deleted')
@@ -80,6 +92,7 @@ options:
         description:
             - How long the module should wait (in seconds) for desired state before returning. Zero means wait as long as necessary.
         default: 0
+
 extends_documentation_fragment:
     - aws
     - ec2
@@ -350,7 +363,36 @@ class EFSConnection(object):
 
         return list(targets)
 
-    def create_file_system(self, name, performance_mode, encrypt, kms_key_id):
+    def supports_provisioned_mode(self):
+        """
+        Ensure boto3 includes provisioned throughput mode feature
+        """
+        return hasattr(self.connection, 'update_file_system')
+
+    def get_throughput_mode(self, **kwargs):
+        """
+        Returns throughput mode for selected EFS instance
+        """
+        info = first_or_default(iterate_all(
+            'FileSystems',
+            self.connection.describe_file_systems,
+            **kwargs
+        ))
+
+        return info and info['ThroughputMode'] or None
+
+    def get_provisioned_throughput_in_mibps(self, **kwargs):
+        """
+        Returns throughput mode for selected EFS instance
+        """
+        info = first_or_default(iterate_all(
+            'FileSystems',
+            self.connection.describe_file_systems,
+            **kwargs
+        ))
+        return info.get('ProvisionedThroughputInMibps', None)
+
+    def create_file_system(self, name, performance_mode, encrypt, kms_key_id, throughput_mode, provisioned_throughput_in_mibps):
         """
          Creates new filesystem with selected name
         """
@@ -363,6 +405,16 @@ class EFSConnection(object):
             params['Encrypted'] = encrypt
         if kms_key_id is not None:
             params['KmsKeyId'] = kms_key_id
+        if throughput_mode:
+            if self.supports_provisioned_mode():
+                params['ThroughputMode'] = throughput_mode
+            else:
+                self.module.fail_json(msg="throughput_mode parameter requires botocore >= 1.10.57")
+        if provisioned_throughput_in_mibps:
+            if self.supports_provisioned_mode():
+                params['ProvisionedThroughputInMibps'] = provisioned_throughput_in_mibps
+            else:
+                self.module.fail_json(msg="provisioned_throughput_in_mibps parameter requires botocore >= 1.10.57")
 
         if state in [self.STATE_DELETING, self.STATE_DELETED]:
             wait_for(
@@ -390,7 +442,39 @@ class EFSConnection(object):
 
         return changed
 
-    def converge_file_system(self, name, tags, purge_tags, targets):
+    def update_file_system(self, name, throughput_mode, provisioned_throughput_in_mibps):
+        """
+        Update filesystem with new throughput settings
+        """
+        changed = False
+        state = self.get_file_system_state(name)
+        if state in [self.STATE_AVAILABLE, self.STATE_CREATING]:
+            fs_id = self.get_file_system_id(name)
+            current_mode = self.get_throughput_mode(FileSystemId=fs_id)
+            current_throughput = self.get_provisioned_throughput_in_mibps(FileSystemId=fs_id)
+            params = dict()
+            if throughput_mode and throughput_mode != current_mode:
+                params['ThroughputMode'] = throughput_mode
+            if provisioned_throughput_in_mibps and provisioned_throughput_in_mibps != current_throughput:
+                params['ProvisionedThroughputInMibps'] = provisioned_throughput_in_mibps
+            if len(params) > 0:
+                wait_for(
+                    lambda: self.get_file_system_state(name),
+                    self.STATE_AVAILABLE,
+                    self.wait_timeout
+                )
+                try:
+                    self.connection.update_file_system(FileSystemId=fs_id, **params)
+                    changed = True
+                except ClientError as e:
+                    self.module.fail_json(msg="Unable to update file system: {0}".format(to_native(e)),
+                                          exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+                except BotoCoreError as e:
+                    self.module.fail_json(msg="Unable to update file system: {0}".format(to_native(e)),
+                                          exception=traceback.format_exc())
+        return changed
+
+    def converge_file_system(self, name, tags, purge_tags, targets, throughput_mode, provisioned_throughput_in_mibps):
         """
          Change attributes (mount targets and tags) of filesystem by name
         """
@@ -620,12 +704,13 @@ def main():
         tags=dict(required=False, type="dict", default={}),
         targets=dict(required=False, type="list", default=[]),
         performance_mode=dict(required=False, type='str', choices=["general_purpose", "max_io"], default="general_purpose"),
+        throughput_mode=dict(required=False, type='str', choices=["bursting", "provisioned"], default=None),
+        provisioned_throughput_in_mibps=dict(required=False, type=float),
         wait=dict(required=False, type="bool", default=False),
         wait_timeout=dict(required=False, type="int", default=0)
     ))
 
     module = AnsibleModule(argument_spec=argument_spec)
-
     if not HAS_BOTO3:
         module.fail_json(msg='boto3 required for this module')
 
@@ -649,16 +734,20 @@ def main():
     kms_key_id = module.params.get('kms_key_id')
     performance_mode = performance_mode_translations[module.params.get('performance_mode')]
     purge_tags = module.params.get('purge_tags')
-    changed = False
+    throughput_mode = module.params.get('throughput_mode')
-
+    provisioned_throughput_in_mibps = module.params.get('provisioned_throughput_in_mibps')
     state = str(module.params.get('state')).lower()
+    changed = False
 
     if state == 'present':
         if not name:
             module.fail_json(msg='Name parameter is required for create')
 
-        changed = connection.create_file_system(name, performance_mode, encrypt, kms_key_id)
+        changed = connection.create_file_system(name, performance_mode, encrypt, kms_key_id, throughput_mode, provisioned_throughput_in_mibps)
-        changed = connection.converge_file_system(name=name, tags=tags, purge_tags=purge_tags, targets=targets) or changed
+        if connection.supports_provisioned_mode():
+            changed = connection.update_file_system(name, throughput_mode, provisioned_throughput_in_mibps) or changed
+        changed = connection.converge_file_system(name=name, tags=tags, purge_tags=purge_tags, targets=targets,
+                                                  throughput_mode=throughput_mode, provisioned_throughput_in_mibps=provisioned_throughput_in_mibps) or changed
         result = first_or_default(connection.get_file_systems(CreationToken=name))
 
     elif state == 'absent':

lib/ansible/modules/cloud/amazon/efs_facts.py

@@ -141,6 +141,16 @@ performance_mode:
     returned: always
     type: str
     sample: "generalPurpose"
+throughput_mode:
+    description: mode of throughput for the file system
+    returned: when botocore >= 1.10.57
+    type: str
+    sample: "bursting"
+provisioned_throughput_in_mibps:
+    description: throughput provisioned in Mibps
+    returned: when botocore >= 1.10.57 and throughput_mode is set to "provisioned"
+    type: float
+    sample: 15.0
 tags:
     description: tags on the efs instance
     returned: always

test/integration/targets/efs/aliases

@@ -1,2 +1,3 @@
 cloud/aws
 unsupported
+efs_facts

test/integration/targets/efs/playbooks/full_test.yml

@@ -0,0 +1,8 @@
+- hosts: localhost
+  connection: local
+
+  vars:
+    resource_prefix: 'ansible-testing'
+
+  roles:
+    - efs

test/integration/targets/efs/playbooks/roles/efs/tasks/main.yml

@@ -66,6 +66,7 @@
         targets:
             - subnet_id: "{{testing_subnet_a.subnet.id}}"
             - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'bursting'
       register: created_efs
 
     # ============================================================
@@ -99,6 +100,7 @@
           - efs_result.ansible_facts.efs[0].encrypted == false
           - efs_result.ansible_facts.efs[0].life_cycle_state == "available"
           - efs_result.ansible_facts.efs[0].performance_mode == "generalPurpose"
+          - efs_result.ansible_facts.efs[0].throughput_mode  == "bursting"
           - efs_result.ansible_facts.efs[0].mount_targets[0].security_groups[0] == vpc_default_sg_id
           - efs_result.ansible_facts.efs[0].mount_targets[1].security_groups[0] == vpc_default_sg_id
 
@@ -161,6 +163,90 @@
     - assert:
         that: "{{efs_result_assertions}}"
 
+    # ============================================================
+    #   Not checking efs_result.efs["throughput_mode"] here as
+    #   Efs with status "life_cycle_state": "updating" might return the previous values
+    - name: Update Efs to use provisioned throughput_mode
+      efs:
+        <<: *aws_connection_info
+        state: present
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+            Name: "{{ resource_prefix }}-test-tag"
+            Purpose: file-storage
+        targets:
+            - subnet_id: "{{testing_subnet_a.subnet.id}}"
+            - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'provisioned'
+        provisioned_throughput_in_mibps: 5.0
+      register: efs_result
+
+    - assert: 
+        that: 
+          - efs_result is changed
+
+    # ============================================================
+    - name: Efs same value for provisioned_throughput_in_mibps
+      efs:
+        <<: *aws_connection_info
+        state: present
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+            Name: "{{ resource_prefix }}-test-tag"
+            Purpose: file-storage
+        targets:
+            - subnet_id: "{{testing_subnet_a.subnet.id}}"
+            - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'provisioned'
+        provisioned_throughput_in_mibps: 5.0
+      register: efs_result
+ 
+    - assert:
+        that: 
+          - efs_result is not changed
+          - efs_result.efs["throughput_mode"]  == "provisioned"
+          - efs_result.efs["provisioned_throughput_in_mibps"] == 5.0
+
+    # ============================================================
+    - name: Efs new value for provisioned_throughput_in_mibps
+      efs:
+        <<: *aws_connection_info
+        state: present
+        name: "{{ resource_prefix }}-test-efs"
+        tags:
+            Name: "{{ resource_prefix }}-test-tag"
+            Purpose: file-storage
+        targets:
+            - subnet_id: "{{testing_subnet_a.subnet.id}}"
+            - subnet_id: "{{testing_subnet_b.subnet.id}}"
+        throughput_mode: 'provisioned'
+        provisioned_throughput_in_mibps: 8.0
+      register: efs_result
+
+    - assert:
+        that:
+          - efs_result is changed
+          - efs_result.efs["provisioned_throughput_in_mibps"] == 8.0
+
+    # ============================================================
+    - name: Check new facts with provisioned mode
+      efs_facts:
+        name: "{{ resource_prefix }}-test-efs"
+        <<: *aws_connection_info
+      register: efs_result
+
+    - set_fact:
+        efs_result_assertions:
+          - efs_result is not changed
+          - efs_result.ansible_facts.efs[0].throughput_mode  == "provisioned"
+          - efs_result.ansible_facts.efs[0].provisioned_throughput_in_mibps == 8.0
+          - (efs_result.ansible_facts.efs | length) == 1
+          - efs_result.ansible_facts.efs[0].creation_token == "{{ resource_prefix }}-test-efs"
+          - efs_result.ansible_facts.efs[0].file_system_id == created_efs.efs.file_system_id
+
+    - assert:
+        that: "{{efs_result_assertions}}"
+
     # ============================================================
     - name: Query unknown EFS by tag
       efs_facts:

test/integration/targets/efs/playbooks/version_fail.yml

@@ -0,0 +1,31 @@
+- hosts: localhost
+  connection: local
+  vars:
+     resource_prefix: 'ansible-testing'
+
+  tasks:
+    - block:
+        - name: set up aws connection info
+          set_fact:
+            aws_connection_info: &aws_connection_info
+              aws_access_key: "{{ aws_access_key }}"
+              aws_secret_key: "{{ aws_secret_key }}"
+              security_token: "{{ security_token }}"
+              region: "{{ aws_region }}"
+          no_log: True
+
+        - name: create efs with provisioned_throughput options (fails gracefully)
+          efs:
+            state: present
+            name: "{{ resource_prefix }}-efs"
+            throughput_mode: 'provisioned'
+            provisioned_throughput_in_mibps: 8.0
+            <<: *aws_connection_info
+          register: efs_provisioned_throughput_creation
+          ignore_errors: yes
+
+        - name: check that graceful error message is returned when creation with throughput_mode and old botocore
+          assert:
+            that:
+              - efs_provisioned_throughput_creation.failed
+              - 'efs_provisioned_throughput_creation.msg == "throughput_mode parameter requires botocore >= 1.10.57"'

test/integration/targets/efs/runme.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# We don't set -u here, due to pypa/virtualenv#150
+set -ex
+MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+# This is needed for the ubuntu1604py3 tests
+# Ubuntu patches virtualenv to make the default python2
+# but for the python3 tests we need virtualenv to use python3
+PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python}
+# Test graceful failure for older versions of botocore
+export ANSIBLE_ROLES_PATH=../
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-less-than-1.10.57"
+source "${MYTMPDIR}/botocore-less-than-1.10.57/bin/activate"
+"${PYTHON}" -m pip install 'botocore<1.10.57' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/version_fail.yml "$@"
+# Run full test suite
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-recent"
+source "${MYTMPDIR}/botocore-recent/bin/activate"
+$PYTHON -m pip install 'botocore>=1.10.57' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"