diff --git a/lib/ansible/modules/cloud/amazon/efs.py b/lib/ansible/modules/cloud/amazon/efs.py index 786ec0f933..69583b1dde 100644 --- a/lib/ansible/modules/cloud/amazon/efs.py +++ b/lib/ansible/modules/cloud/amazon/efs.py @@ -70,6 +70,18 @@ options: - ip_address - Optional. A valid IPv4 address within the address range of the specified subnet. - security_groups - Optional. List of security group IDs, of the form 'sg-xxxxxxxx'. These must be for the same VPC as subnet specified This data may be modified for existing EFS using state 'present' and new list of mount targets." + throughput_mode: + description: + - The throughput_mode for the file system to be created. + - Requires botocore >= 1.10.57 + choices: ['bursting', 'provisioned'] + version_added: 2.8 + provisioned_throughput_in_mibps: + description: + - If the throughput_mode is provisioned, select the amount of throughput to provisioned in Mibps. + - Requires botocore >= 1.10.57 + type: float + version_added: 2.8 wait: description: - "In case of 'present' state should wait for EFS 'available' life cycle state (of course, if current state not 'deleting' or 'deleted') @@ -80,6 +92,7 @@ options: description: - How long the module should wait (in seconds) for desired state before returning. Zero means wait as long as necessary. default: 0 + extends_documentation_fragment: - aws - ec2 @@ -350,7 +363,36 @@ class EFSConnection(object): return list(targets) - def create_file_system(self, name, performance_mode, encrypt, kms_key_id): + def supports_provisioned_mode(self): + """ + Ensure boto3 includes provisioned throughput mode feature + """ + return hasattr(self.connection, 'update_file_system') + + def get_throughput_mode(self, **kwargs): + """ + Returns throughput mode for selected EFS instance + """ + info = first_or_default(iterate_all( + 'FileSystems', + self.connection.describe_file_systems, + **kwargs + )) + + return info and info['ThroughputMode'] or None + + def get_provisioned_throughput_in_mibps(self, **kwargs): + """ + Returns throughput mode for selected EFS instance + """ + info = first_or_default(iterate_all( + 'FileSystems', + self.connection.describe_file_systems, + **kwargs + )) + return info.get('ProvisionedThroughputInMibps', None) + + def create_file_system(self, name, performance_mode, encrypt, kms_key_id, throughput_mode, provisioned_throughput_in_mibps): """ Creates new filesystem with selected name """ @@ -363,6 +405,16 @@ class EFSConnection(object): params['Encrypted'] = encrypt if kms_key_id is not None: params['KmsKeyId'] = kms_key_id + if throughput_mode: + if self.supports_provisioned_mode(): + params['ThroughputMode'] = throughput_mode + else: + self.module.fail_json(msg="throughput_mode parameter requires botocore >= 1.10.57") + if provisioned_throughput_in_mibps: + if self.supports_provisioned_mode(): + params['ProvisionedThroughputInMibps'] = provisioned_throughput_in_mibps + else: + self.module.fail_json(msg="provisioned_throughput_in_mibps parameter requires botocore >= 1.10.57") if state in [self.STATE_DELETING, self.STATE_DELETED]: wait_for( @@ -390,7 +442,39 @@ class EFSConnection(object): return changed - def converge_file_system(self, name, tags, purge_tags, targets): + def update_file_system(self, name, throughput_mode, provisioned_throughput_in_mibps): + """ + Update filesystem with new throughput settings + """ + changed = False + state = self.get_file_system_state(name) + if state in [self.STATE_AVAILABLE, self.STATE_CREATING]: + fs_id = self.get_file_system_id(name) + current_mode = self.get_throughput_mode(FileSystemId=fs_id) + current_throughput = self.get_provisioned_throughput_in_mibps(FileSystemId=fs_id) + params = dict() + if throughput_mode and throughput_mode != current_mode: + params['ThroughputMode'] = throughput_mode + if provisioned_throughput_in_mibps and provisioned_throughput_in_mibps != current_throughput: + params['ProvisionedThroughputInMibps'] = provisioned_throughput_in_mibps + if len(params) > 0: + wait_for( + lambda: self.get_file_system_state(name), + self.STATE_AVAILABLE, + self.wait_timeout + ) + try: + self.connection.update_file_system(FileSystemId=fs_id, **params) + changed = True + except ClientError as e: + self.module.fail_json(msg="Unable to update file system: {0}".format(to_native(e)), + exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) + except BotoCoreError as e: + self.module.fail_json(msg="Unable to update file system: {0}".format(to_native(e)), + exception=traceback.format_exc()) + return changed + + def converge_file_system(self, name, tags, purge_tags, targets, throughput_mode, provisioned_throughput_in_mibps): """ Change attributes (mount targets and tags) of filesystem by name """ @@ -620,12 +704,13 @@ def main(): tags=dict(required=False, type="dict", default={}), targets=dict(required=False, type="list", default=[]), performance_mode=dict(required=False, type='str', choices=["general_purpose", "max_io"], default="general_purpose"), + throughput_mode=dict(required=False, type='str', choices=["bursting", "provisioned"], default=None), + provisioned_throughput_in_mibps=dict(required=False, type=float), wait=dict(required=False, type="bool", default=False), wait_timeout=dict(required=False, type="int", default=0) )) module = AnsibleModule(argument_spec=argument_spec) - if not HAS_BOTO3: module.fail_json(msg='boto3 required for this module') @@ -649,16 +734,20 @@ def main(): kms_key_id = module.params.get('kms_key_id') performance_mode = performance_mode_translations[module.params.get('performance_mode')] purge_tags = module.params.get('purge_tags') - changed = False - + throughput_mode = module.params.get('throughput_mode') + provisioned_throughput_in_mibps = module.params.get('provisioned_throughput_in_mibps') state = str(module.params.get('state')).lower() + changed = False if state == 'present': if not name: module.fail_json(msg='Name parameter is required for create') - changed = connection.create_file_system(name, performance_mode, encrypt, kms_key_id) - changed = connection.converge_file_system(name=name, tags=tags, purge_tags=purge_tags, targets=targets) or changed + changed = connection.create_file_system(name, performance_mode, encrypt, kms_key_id, throughput_mode, provisioned_throughput_in_mibps) + if connection.supports_provisioned_mode(): + changed = connection.update_file_system(name, throughput_mode, provisioned_throughput_in_mibps) or changed + changed = connection.converge_file_system(name=name, tags=tags, purge_tags=purge_tags, targets=targets, + throughput_mode=throughput_mode, provisioned_throughput_in_mibps=provisioned_throughput_in_mibps) or changed result = first_or_default(connection.get_file_systems(CreationToken=name)) elif state == 'absent': diff --git a/lib/ansible/modules/cloud/amazon/efs_facts.py b/lib/ansible/modules/cloud/amazon/efs_facts.py index af697ecf9e..0db4fb0c30 100644 --- a/lib/ansible/modules/cloud/amazon/efs_facts.py +++ b/lib/ansible/modules/cloud/amazon/efs_facts.py @@ -141,6 +141,16 @@ performance_mode: returned: always type: str sample: "generalPurpose" +throughput_mode: + description: mode of throughput for the file system + returned: when botocore >= 1.10.57 + type: str + sample: "bursting" +provisioned_throughput_in_mibps: + description: throughput provisioned in Mibps + returned: when botocore >= 1.10.57 and throughput_mode is set to "provisioned" + type: float + sample: 15.0 tags: description: tags on the efs instance returned: always diff --git a/test/integration/targets/efs_facts/aliases b/test/integration/targets/efs/aliases similarity index 68% rename from test/integration/targets/efs_facts/aliases rename to test/integration/targets/efs/aliases index 5692719518..8b745e200d 100644 --- a/test/integration/targets/efs_facts/aliases +++ b/test/integration/targets/efs/aliases @@ -1,2 +1,3 @@ cloud/aws unsupported +efs_facts diff --git a/test/integration/targets/efs/playbooks/full_test.yml b/test/integration/targets/efs/playbooks/full_test.yml new file mode 100644 index 0000000000..d31f5a552c --- /dev/null +++ b/test/integration/targets/efs/playbooks/full_test.yml @@ -0,0 +1,8 @@ +- hosts: localhost + connection: local + + vars: + resource_prefix: 'ansible-testing' + + roles: + - efs diff --git a/test/integration/targets/efs_facts/tasks/main.yml b/test/integration/targets/efs/playbooks/roles/efs/tasks/main.yml similarity index 70% rename from test/integration/targets/efs_facts/tasks/main.yml rename to test/integration/targets/efs/playbooks/roles/efs/tasks/main.yml index efdd0bdef0..15b644fcbf 100644 --- a/test/integration/targets/efs_facts/tasks/main.yml +++ b/test/integration/targets/efs/playbooks/roles/efs/tasks/main.yml @@ -66,6 +66,7 @@ targets: - subnet_id: "{{testing_subnet_a.subnet.id}}" - subnet_id: "{{testing_subnet_b.subnet.id}}" + throughput_mode: 'bursting' register: created_efs # ============================================================ @@ -99,6 +100,7 @@ - efs_result.ansible_facts.efs[0].encrypted == false - efs_result.ansible_facts.efs[0].life_cycle_state == "available" - efs_result.ansible_facts.efs[0].performance_mode == "generalPurpose" + - efs_result.ansible_facts.efs[0].throughput_mode == "bursting" - efs_result.ansible_facts.efs[0].mount_targets[0].security_groups[0] == vpc_default_sg_id - efs_result.ansible_facts.efs[0].mount_targets[1].security_groups[0] == vpc_default_sg_id @@ -161,6 +163,90 @@ - assert: that: "{{efs_result_assertions}}" + # ============================================================ + # Not checking efs_result.efs["throughput_mode"] here as + # Efs with status "life_cycle_state": "updating" might return the previous values + - name: Update Efs to use provisioned throughput_mode + efs: + <<: *aws_connection_info + state: present + name: "{{ resource_prefix }}-test-efs" + tags: + Name: "{{ resource_prefix }}-test-tag" + Purpose: file-storage + targets: + - subnet_id: "{{testing_subnet_a.subnet.id}}" + - subnet_id: "{{testing_subnet_b.subnet.id}}" + throughput_mode: 'provisioned' + provisioned_throughput_in_mibps: 5.0 + register: efs_result + + - assert: + that: + - efs_result is changed + + # ============================================================ + - name: Efs same value for provisioned_throughput_in_mibps + efs: + <<: *aws_connection_info + state: present + name: "{{ resource_prefix }}-test-efs" + tags: + Name: "{{ resource_prefix }}-test-tag" + Purpose: file-storage + targets: + - subnet_id: "{{testing_subnet_a.subnet.id}}" + - subnet_id: "{{testing_subnet_b.subnet.id}}" + throughput_mode: 'provisioned' + provisioned_throughput_in_mibps: 5.0 + register: efs_result + + - assert: + that: + - efs_result is not changed + - efs_result.efs["throughput_mode"] == "provisioned" + - efs_result.efs["provisioned_throughput_in_mibps"] == 5.0 + + # ============================================================ + - name: Efs new value for provisioned_throughput_in_mibps + efs: + <<: *aws_connection_info + state: present + name: "{{ resource_prefix }}-test-efs" + tags: + Name: "{{ resource_prefix }}-test-tag" + Purpose: file-storage + targets: + - subnet_id: "{{testing_subnet_a.subnet.id}}" + - subnet_id: "{{testing_subnet_b.subnet.id}}" + throughput_mode: 'provisioned' + provisioned_throughput_in_mibps: 8.0 + register: efs_result + + - assert: + that: + - efs_result is changed + - efs_result.efs["provisioned_throughput_in_mibps"] == 8.0 + + # ============================================================ + - name: Check new facts with provisioned mode + efs_facts: + name: "{{ resource_prefix }}-test-efs" + <<: *aws_connection_info + register: efs_result + + - set_fact: + efs_result_assertions: + - efs_result is not changed + - efs_result.ansible_facts.efs[0].throughput_mode == "provisioned" + - efs_result.ansible_facts.efs[0].provisioned_throughput_in_mibps == 8.0 + - (efs_result.ansible_facts.efs | length) == 1 + - efs_result.ansible_facts.efs[0].creation_token == "{{ resource_prefix }}-test-efs" + - efs_result.ansible_facts.efs[0].file_system_id == created_efs.efs.file_system_id + + - assert: + that: "{{efs_result_assertions}}" + # ============================================================ - name: Query unknown EFS by tag efs_facts: diff --git a/test/integration/targets/efs/playbooks/version_fail.yml b/test/integration/targets/efs/playbooks/version_fail.yml new file mode 100644 index 0000000000..a8b923e9a7 --- /dev/null +++ b/test/integration/targets/efs/playbooks/version_fail.yml @@ -0,0 +1,31 @@ +- hosts: localhost + connection: local + vars: + resource_prefix: 'ansible-testing' + + tasks: + - block: + - name: set up aws connection info + set_fact: + aws_connection_info: &aws_connection_info + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token }}" + region: "{{ aws_region }}" + no_log: True + + - name: create efs with provisioned_throughput options (fails gracefully) + efs: + state: present + name: "{{ resource_prefix }}-efs" + throughput_mode: 'provisioned' + provisioned_throughput_in_mibps: 8.0 + <<: *aws_connection_info + register: efs_provisioned_throughput_creation + ignore_errors: yes + + - name: check that graceful error message is returned when creation with throughput_mode and old botocore + assert: + that: + - efs_provisioned_throughput_creation.failed + - 'efs_provisioned_throughput_creation.msg == "throughput_mode parameter requires botocore >= 1.10.57"' diff --git a/test/integration/targets/efs/runme.sh b/test/integration/targets/efs/runme.sh new file mode 100755 index 0000000000..6db90bbee1 --- /dev/null +++ b/test/integration/targets/efs/runme.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# We don't set -u here, due to pypa/virtualenv#150 +set -ex +MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') +trap 'rm -rf "${MYTMPDIR}"' EXIT +# This is needed for the ubuntu1604py3 tests +# Ubuntu patches virtualenv to make the default python2 +# but for the python3 tests we need virtualenv to use python3 +PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python} +# Test graceful failure for older versions of botocore +export ANSIBLE_ROLES_PATH=../ +virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-less-than-1.10.57" +source "${MYTMPDIR}/botocore-less-than-1.10.57/bin/activate" +"${PYTHON}" -m pip install 'botocore<1.10.57' boto3 +ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/version_fail.yml "$@" +# Run full test suite +virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-recent" +source "${MYTMPDIR}/botocore-recent/bin/activate" +$PYTHON -m pip install 'botocore>=1.10.57' boto3 +ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"