1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2024-09-14 20:13:21 +02:00

[cloud] Add more configurable backoff implementations to CloudRetry/AWSRetry (#27251)

This commit is contained in:
mzizzi 2017-08-08 08:56:46 -04:00 committed by Ryan Brown
parent 11af034255
commit 4648dc9702
3 changed files with 176 additions and 17 deletions

View file

@ -31,10 +31,15 @@ The 'cloud' module provides the following common classes:
provide a backoff/retry decorator based on status codes.
- Example using the AWSRetry class which inherits from CloudRetry.
@AWSRetry.retry(tries=20, delay=2, backoff=2)
@AWSRetry.exponential_backoff(retries=10, delay=3)
get_ec2_security_group_ids_from_names()
@AWSRetry.jittered_backoff()
get_ec2_security_group_ids_from_names()
"""
import random
from functools import wraps
import syslog
import time
@ -42,6 +47,60 @@ import time
from ansible.module_utils.pycompat24 import get_exception
def _exponential_backoff(retries=10, delay=2, backoff=2, max_delay=60):
""" Customizable exponential backoff strategy.
Args:
retries (int): Maximum number of times to retry a request.
delay (float): Initial (base) delay.
backoff (float): base of the exponent to use for exponential
backoff.
max_delay (int): Optional. If provided each delay generated is capped
at this amount. Defaults to 60 seconds.
Returns:
Callable that returns a generator. This generator yields durations in
seconds to be used as delays for an exponential backoff strategy.
Usage:
>>> backoff = _exponential_backoff()
>>> backoff
<function backoff_backoff at 0x7f0d939facf8>
>>> list(backoff())
[2, 4, 8, 16, 32, 60, 60, 60, 60, 60]
"""
def backoff_gen():
for retry in range(0, retries):
sleep = delay * backoff ** retry
yield sleep if max_delay is None else min(sleep, max_delay)
return backoff_gen
def _full_jitter_backoff(retries=10, delay=3, max_delay=60, _random=random):
""" Implements the "Full Jitter" backoff strategy described here
https://www.awsarchitectureblog.com/2015/03/backoff.html
Args:
retries (int): Maximum number of times to retry a request.
delay (float): Approximate number of seconds to sleep for the first
retry.
max_delay (int): The maximum number of seconds to sleep for any retry.
_random (random.Random or None): Makes this generator testable by
allowing developers to explicitly pass in the a seeded Random.
Returns:
Callable that returns a generator. This generator yields durations in
seconds to be used as delays for a full jitter backoff strategy.
Usage:
>>> backoff = _full_jitter_backoff(retries=5)
>>> backoff
<function backoff_backoff at 0x7f0d939facf8>
>>> list(backoff())
[3, 6, 5, 23, 38]
>>> list(backoff())
[2, 1, 6, 6, 31]
"""
def backoff_gen():
for retry in range(0, retries):
yield _random.randint(0, min(max_delay, delay * 2 ** retry))
return backoff_gen
class CloudRetry(object):
""" CloudRetry can be used by any cloud provider, in order to implement a
backoff algorithm/retry effect based on Status Code from Exceptions.
@ -67,22 +126,18 @@ class CloudRetry(object):
pass
@classmethod
def backoff(cls, tries=10, delay=3, backoff=1.1):
""" Retry calling the Cloud decorated function using an exponential backoff.
Kwargs:
tries (int): Number of times to try (not retry) before giving up
default=10
delay (int): Initial delay between retries in seconds
default=3
backoff (int): backoff multiplier e.g. value of 2 will double the delay each retry
default=2
def _backoff(cls, backoff_strategy):
""" Retry calling the Cloud decorated function using the provided
backoff strategy.
Args:
backoff_strategy (callable): Callable that returns a generator. The
generator should yield sleep times for each retry of the decorated
function.
"""
def deco(f):
@wraps(f)
def retry_func(*args, **kwargs):
max_tries, max_delay = tries, delay
while max_tries > 1:
for delay in backoff_strategy():
try:
return f(*args, **kwargs)
except Exception:
@ -90,11 +145,9 @@ class CloudRetry(object):
if isinstance(e, cls.base_class):
response_code = cls.status_code_from_exception(e)
if cls.found(response_code):
msg = "{0}: Retrying in {1} seconds...".format(str(e), max_delay)
msg = "{0}: Retrying in {1} seconds...".format(str(e), delay)
syslog.syslog(syslog.LOG_INFO, msg)
time.sleep(max_delay)
max_tries -= 1
max_delay *= backoff
time.sleep(delay)
else:
# Return original exception if exception is not a ClientError
raise e
@ -106,3 +159,62 @@ class CloudRetry(object):
return retry_func # true decorator
return deco
@classmethod
def exponential_backoff(cls, retries=10, delay=3, backoff=2, max_delay=60):
"""
Retry calling the Cloud decorated function using an exponential backoff.
Kwargs:
retries (int): Number of times to retry a failed request before giving up
default=10
delay (int or float): Initial delay between retries in seconds
default=3
backoff (int or float): backoff multiplier e.g. value of 2 will
double the delay each retry
default=1.1
max_delay (int or None): maximum amount of time to wait between retries.
default=60
"""
return cls._backoff(_exponential_backoff(
retries=retries, delay=delay, backoff=backoff, max_delay=max_delay))
@classmethod
def jittered_backoff(cls, retries=10, delay=3, max_delay=60):
"""
Retry calling the Cloud decorated function using a jittered backoff
strategy. More on this strategy here:
https://www.awsarchitectureblog.com/2015/03/backoff.html
Kwargs:
retries (int): Number of times to retry a failed request before giving up
default=10
delay (int): Initial delay between retries in seconds
default=3
max_delay (int): maximum amount of time to wait between retries.
default=60
"""
return cls._backoff(_full_jitter_backoff(
retries=retries, delay=delay, max_delay=max_delay))
@classmethod
def backoff(cls, tries=10, delay=3, backoff=1.1):
"""
Retry calling the Cloud decorated function using an exponential backoff.
Compatibility for the original implementation of CloudRetry.backoff that
did not provide configurable backoff strategies. Developers should use
CloudRetry.exponential_backoff instead.
Kwargs:
tries (int): Number of times to try (not retry) before giving up
default=10
delay (int or float): Initial delay between retries in seconds
default=3
backoff (int or float): backoff multiplier e.g. value of 2 will
double the delay each retry
default=1.1
"""
return cls.exponential_backoff(
retries=tries - 1, delay=delay, backoff=backoff, max_delay=None)

View file

@ -0,0 +1,47 @@
import random
from ansible.compat.tests import unittest
from ansible.module_utils.cloud import _exponential_backoff, \
_full_jitter_backoff
class ExponentialBackoffStrategyTestCase(unittest.TestCase):
def test_no_retries(self):
strategy = _exponential_backoff(retries=0)
result = list(strategy())
self.assertEquals(result, [], 'list should be empty')
def test_exponential_backoff(self):
strategy = _exponential_backoff(retries=5, delay=1, backoff=2)
result = list(strategy())
self.assertEquals(result, [1, 2, 4, 8, 16])
def test_max_delay(self):
strategy = _exponential_backoff(retries=7, delay=1, backoff=2, max_delay=60)
result = list(strategy())
self.assertEquals(result, [1, 2, 4, 8, 16, 32, 60])
def test_max_delay_none(self):
strategy = _exponential_backoff(retries=7, delay=1, backoff=2, max_delay=None)
result = list(strategy())
self.assertEquals(result, [1, 2, 4, 8, 16, 32, 64])
class FullJitterBackoffStrategyTestCase(unittest.TestCase):
def test_no_retries(self):
strategy = _full_jitter_backoff(retries=0)
result = list(strategy())
self.assertEquals(result, [], 'list should be empty')
def test_full_jitter(self):
retries = 5
seed = 1
r = random.Random(seed)
expected = [r.randint(0, 2**i) for i in range(0, retries)]
strategy = _full_jitter_backoff(
retries=retries, delay=1, _random=random.Random(seed))
result = list(strategy())
self.assertEquals(result, expected)