From 587ab3341530572b80d6d8213633ee3fe3cc35dd Mon Sep 17 00:00:00 2001 From: mzizzi Date: Wed, 23 Aug 2017 11:38:19 -0400 Subject: [PATCH] [cloud] Add jittered backoff+retries to CloudFormation module (#27907) --- .../modules/cloud/amazon/cloudformation.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/ansible/modules/cloud/amazon/cloudformation.py b/lib/ansible/modules/cloud/amazon/cloudformation.py index 6fb22ad6f5..f8f04ac70d 100644 --- a/lib/ansible/modules/cloud/amazon/cloudformation.py +++ b/lib/ansible/modules/cloud/amazon/cloudformation.py @@ -417,13 +417,10 @@ def stack_operation(cfn, stack_name, operation): time.sleep(5) return {'failed': True, 'output':'Failed for unknown reasons.'} -@AWSRetry.backoff(tries=3, delay=5) -def describe_stacks(cfn, stack_name): - return cfn.describe_stacks(StackName=stack_name) def get_stack_facts(cfn, stack_name): try: - stack_response = describe_stacks(cfn, stack_name) + stack_response = cfn.describe_stacks(StackName=stack_name) stack_info = stack_response['Stacks'][0] #except AmazonCloudFormationException as e: except (botocore.exceptions.ValidationError,botocore.exceptions.ClientError) as err: @@ -509,6 +506,18 @@ def main(): except botocore.exceptions.NoCredentialsError as e: module.fail_json(msg=boto_exception(e)) + # Wrap the cloudformation client methods that this module uses with + # automatic backoff / retry for throttling error codes + backoff_wrapper = AWSRetry.jittered_backoff(retries=10, delay=3, max_delay=30) + cfn.describe_stack_events = backoff_wrapper(cfn.describe_stack_events) + cfn.create_stack = backoff_wrapper(cfn.create_stack) + cfn.list_change_sets = backoff_wrapper(cfn.list_change_sets) + cfn.create_change_set = backoff_wrapper(cfn.create_change_set) + cfn.update_stack = backoff_wrapper(cfn.update_stack) + cfn.describe_stacks = backoff_wrapper(cfn.describe_stacks) + cfn.list_stack_resources = backoff_wrapper(cfn.list_stack_resources) + cfn.delete_stack = backoff_wrapper(cfn.delete_stack) + stack_info = get_stack_facts(cfn, stack_params['StackName']) if state == 'present':