From 24826bdc3dd5aa08130fd347517acdc13d610b61 Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Fri, 8 Jan 2016 16:49:48 +0800 Subject: [PATCH 01/11] add aws_expo to support Full Jitter --- README.md | 68 ++++++++++++++++++++++++++++ backoff.py | 22 +++++++++ backoff_tests.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+) diff --git a/README.md b/README.md index 37c539e..a77667a 100644 --- a/README.md +++ b/README.md @@ -148,3 +148,71 @@ set the logger level to INFO: logging.getLogger('backoff').setLevel(logging.INFO) +## Examples for Full Jitter + +*Full Jitter algorithm comes from [AWS Blog](http://www.awsarchitectureblog.com/2015/03/backoff.html), basically usage is almost identical to aforementioned examples, hence the unique difference would be described in this session.* + +### @backoff.on_exception + +The ``on_exception`` decorator is used to retry when a specified exception is raised. Here's an example using AWS exponential backoff when any requests exception is raised: + + @backoff.on_exception(backoff.aws_expo, + jitter=lambda: 0, + requests.exceptions.RequestException, + max_tries=8) + def get_url(url): + return requests.get(url) + +To take advantage of Full Jitter, you may just specify ``backoff.aws_expo`` and it should work as expected + + sleep = random_between(0, min(max_value, base * 2 ** attempt)) + +Actually both ``on_exception`` and ``on_predicate`` does slightly jitter as well by default, the idea is adding additional random milliseconds. Since ``aws_expo`` includes jitter, you are able to disable default random jitter by specifying ``jitter=lambda: 0``. + +## Make sure you are ready to 'commit' + +### Virtual Python Environment builder + +``virtualenv`` is a tool to create isolated testing environments which could prevent from pollution. + + # Install virtualenv + $ sudo pip install virtualenv + + # Create virtual environment + $ virtualenv mytest + + # Activate virtual environment + $ cd mytest + $ source bin/activate + +### Python style guide checker + + (mytest)$ sudo pip install pep8 + +### Passive checker of Python programs + + (mytest)$ sudo pip install pyflakes + +### Pytest plugin for measuring coverage + + (mytest)$ sudo pip install pytest-cov + +### Secure quality for your changes + + # Switch to backoff.py and backoff_tests.py folder + (mytest)$ make check + + ============================= test session starts ============================== + platform darwin -- Python 2.7.10, pytest-2.8.5, py-1.4.31, pluggy-0.3.1 + rootdir: /Github/backoff, inifile: + plugins: cov-2.2.0 + collected 16 items + + backoff_tests.py ................ + --------------- coverage: platform darwin, python 2.7.10-final-0 --------------- + Name Stmts Miss Cover Missing + ------------------------------------------ + backoff.py 123 0 100% + + ========================== 25 passed in 0.18 seconds =========================== + diff --git a/backoff.py b/backoff.py index b189fe2..5ba0316 100644 --- a/backoff.py +++ b/backoff.py @@ -171,6 +171,28 @@ def emit(self, record): logger.setLevel(logging.ERROR) +def aws_expo(base=0.5, max_value=None, full_jitter=True): + """Generator for exponential decay base on AWS Blog. + Ref: http://www.awsarchitectureblog.com/2015/03/backoff.html + + Args: + base: The mathematical base of the exponentiation operation + max_value: The maximum value to yield. Once the value in the + true exponential sequence exceeds this, the value + of max_value will forever after be yielded. + full_jitter: Enable Full Jitter algorithm, the default value + is true + """ + n = 0 + while True: + a = base * 2 ** n + if max_value is None or a < max_value: + yield random.uniform(0, a) if full_jitter else a + n += 1 + else: + yield random.uniform(0, max_value) if full_jitter else max_value + + def expo(base=2, max_value=None): """Generator for exponential decay. diff --git a/backoff_tests.py b/backoff_tests.py index 8fa94e3..7466fb3 100644 --- a/backoff_tests.py +++ b/backoff_tests.py @@ -6,18 +6,45 @@ import pytest +def test_aws_expo(): + gen = backoff.aws_expo(full_jitter=False) + for i in range(9): + assert 0.5 * 2 ** i == next(gen) + + def test_expo(): gen = backoff.expo() for i in range(9): assert 2 ** i == next(gen) +def test_aws_expo_base3(): + gen = backoff.aws_expo(base=3, full_jitter=False) + for i in range(9): + assert 3 * 2 ** i == next(gen) + + def test_expo_base3(): gen = backoff.expo(base=3) for i in range(9): assert 3 ** i == next(gen) +def test_aws_expo_max_value(): + gen = backoff.aws_expo(base=0.5, max_value=2 ** 4, full_jitter=False) + expected = [0.5, 1, 2, 4, 8, 16, 16, 16] + for expect in expected: + assert expect == next(gen) + + +def test_aws_expo_max_random_value(): + gen = backoff.aws_expo(base=0.5, max_value=2) + expected = [0.5, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] + + for expect in expected: + assert expect >= next(gen) + + def test_expo_max_value(): gen = backoff.expo(max_value=2 ** 4) expected = [1, 2, 4, 8, 16, 16, 16] @@ -45,6 +72,22 @@ def test_constant(): assert 3 == next(gen) +def test_on_predicate_aws(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + @backoff.on_predicate(backoff.aws_expo, + jitter=lambda: 0) + def return_true(log, n): + val = (len(log) == n - 1) + log.append(val) + return val + + log = [] + ret = return_true(log, 3) + assert ret is True + assert 3 == len(log) + + def test_on_predicate(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -60,6 +103,21 @@ def return_true(log, n): assert 3 == len(log) +def test_on_predicate_aws_max_tries(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + @backoff.on_predicate(backoff.aws_expo, max_tries=3) + def return_true(log, n): + val = (len(log) == n) + log.append(val) + return val + + log = [] + ret = return_true(log, 10) + assert ret is False + assert 3 == len(log) + + def test_on_predicate_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -75,6 +133,22 @@ def return_true(log, n): assert 3 == len(log) +def test_on_exception_aws(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + @backoff.on_exception(backoff.aws_expo, KeyError) + def keyerror_then_true(log, n): + if len(log) == n: + return True + e = KeyError() + log.append(e) + raise e + + log = [] + assert keyerror_then_true(log, 3) is True + assert 3 == len(log) + + def test_on_exception(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -91,6 +165,27 @@ def keyerror_then_true(log, n): assert 3 == len(log) +def test_on_exception_aws_tuple(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + @backoff.on_exception(backoff.aws_expo, (KeyError, ValueError)) + def keyerror_valueerror_then_true(log): + if len(log) == 2: + return True + if len(log) == 0: + e = KeyError() + if len(log) == 1: + e = ValueError() + log.append(e) + raise e + + log = [] + assert keyerror_valueerror_then_true(log) is True + assert 2 == len(log) + assert isinstance(log[0], KeyError) + assert isinstance(log[1], ValueError) + + def test_on_exception_tuple(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -112,6 +207,24 @@ def keyerror_valueerror_then_true(log): assert isinstance(log[1], ValueError) +def test_on_exception_aws_max_tries(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + @backoff.on_exception(backoff.aws_expo, KeyError, max_tries=3) + def keyerror_then_true(log, n, foo=None): + if len(log) == n: + return True + e = KeyError() + log.append(e) + raise e + + log = [] + with pytest.raises(KeyError): + keyerror_then_true(log, 10, foo="bar") + + assert 3 == len(log) + + def test_on_exception_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) From d4bdbb4d6fdda5cd8cb82fb089732e235c4c95dd Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Wed, 20 Jan 2016 11:30:36 +0800 Subject: [PATCH 02/11] refactor code to support random, full, and equal jitter --- backoff.py | 70 +++++++------- backoff_tests.py | 236 ++++++++++++++++++++++++----------------------- 2 files changed, 154 insertions(+), 152 deletions(-) diff --git a/backoff.py b/backoff.py index 5ba0316..2e2fecc 100644 --- a/backoff.py +++ b/backoff.py @@ -171,29 +171,7 @@ def emit(self, record): logger.setLevel(logging.ERROR) -def aws_expo(base=0.5, max_value=None, full_jitter=True): - """Generator for exponential decay base on AWS Blog. - Ref: http://www.awsarchitectureblog.com/2015/03/backoff.html - - Args: - base: The mathematical base of the exponentiation operation - max_value: The maximum value to yield. Once the value in the - true exponential sequence exceeds this, the value - of max_value will forever after be yielded. - full_jitter: Enable Full Jitter algorithm, the default value - is true - """ - n = 0 - while True: - a = base * 2 ** n - if max_value is None or a < max_value: - yield random.uniform(0, a) if full_jitter else a - n += 1 - else: - yield random.uniform(0, max_value) if full_jitter else max_value - - -def expo(base=2, max_value=None): +def expo(init_value=1, base=2, max_value=None): """Generator for exponential decay. Args: @@ -204,7 +182,7 @@ def expo(base=2, max_value=None): """ n = 0 while True: - a = base ** n + a = init_value * base ** n if max_value is None or a < max_value: yield a n += 1 @@ -240,10 +218,22 @@ def constant(interval=1): yield interval +def random_jitter(value): + return value+random.random() + + +def full_jitter(value): + return random.uniform(0, value) + + +def equal_jitter(value): + return (value/2.0) + (random.uniform(0, value/2.0)) + + def on_predicate(wait_gen, predicate=operator.not_, max_tries=None, - jitter=random.random, + jitter=None, on_success=None, on_backoff=None, on_giveup=None, @@ -261,11 +251,10 @@ def on_predicate(wait_gen, up. In the case of failure, the result of the last attempt will be returned. The default value of None means their is no limit to the number of tries. - jitter: Callable returning an offset in seconds to add to the - value yielded by wait_gen. When used with the default - random function, this staggers wait times a random number - of milliseconds to help spread out load in the case that - there are multiple simultaneous retries occuring. + jitter: Callable returning an offset to the value yielded by wait_gen. + This staggers wait times a random number of milliseconds to help + spread out load in the case that there are multiple simultaneous + retries occuring. on_success: Callable (or iterable of callables) with a unary signature to be called in the event of success. The parameter is a dict containing details about the invocation. @@ -303,7 +292,10 @@ def retry(*args, **kwargs): 'value': ret}) break - seconds = next(wait) + jitter() + if jitter is not None: + seconds = jitter(next(wait)) + else: + seconds = next(wait) for hdlr in backoff_hdlrs: hdlr({'target': target, @@ -335,7 +327,7 @@ def retry(*args, **kwargs): def on_exception(wait_gen, exception, max_tries=None, - jitter=random.random, + jitter=None, on_success=None, on_backoff=None, on_giveup=None, @@ -351,11 +343,10 @@ def on_exception(wait_gen, up. Once exhausted, the exception will be allowed to escape. The default value of None means their is no limit to the number of tries. - jitter: Callable returning an offset in seconds to add to the - value yielded by wait_gen. When used with the default - random function, this staggers wait times a random number - of milliseconds to help spread out load in the case that - there are multiple simultaneous retries occuring. + jitter: Callable returning an offset to the value yielded by wait_gen. + This staggers wait times a random number of milliseconds to help + spread out load in the case that there are multiple simultaneous + retries occuring. on_success: Callable (or iterable of callables) with a unary signature to be called in the event of success. The parameter is a dict containing details about the invocation. @@ -393,7 +384,10 @@ def retry(*args, **kwargs): 'tries': tries}) raise - seconds = next(wait) + jitter() + if jitter is not None: + seconds = jitter(next(wait)) + else: + seconds = next(wait) for hdlr in backoff_hdlrs: hdlr({'target': target, diff --git a/backoff_tests.py b/backoff_tests.py index 7466fb3..58f7a35 100644 --- a/backoff_tests.py +++ b/backoff_tests.py @@ -6,10 +6,20 @@ import pytest -def test_aws_expo(): - gen = backoff.aws_expo(full_jitter=False) - for i in range(9): - assert 0.5 * 2 ** i == next(gen) +def test_full_jitter(): + for input in range(100): + for i in range(100): + jitter = backoff.full_jitter(input) + assert jitter >= 0 + assert jitter <= input + + +def test_equal_jitter(): + for input in range(100): + for i in range(100): + jitter = backoff.equal_jitter(input) + assert jitter >= input/2.0 + assert jitter <= input def test_expo(): @@ -18,31 +28,22 @@ def test_expo(): assert 2 ** i == next(gen) -def test_aws_expo_base3(): - gen = backoff.aws_expo(base=3, full_jitter=False) - for i in range(9): - assert 3 * 2 ** i == next(gen) - - def test_expo_base3(): gen = backoff.expo(base=3) for i in range(9): assert 3 ** i == next(gen) -def test_aws_expo_max_value(): - gen = backoff.aws_expo(base=0.5, max_value=2 ** 4, full_jitter=False) - expected = [0.5, 1, 2, 4, 8, 16, 16, 16] - for expect in expected: - assert expect == next(gen) - +def test_expo_init3(): + gen = backoff.expo(init_value=3) + for i in range(9): + assert 3 * 2 ** i == next(gen) -def test_aws_expo_max_random_value(): - gen = backoff.aws_expo(base=0.5, max_value=2) - expected = [0.5, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] - for expect in expected: - assert expect >= next(gen) +def test_expo_base3_init5(): + gen = backoff.expo(base=3, init_value=5) + for i in range(9): + assert 5 * 3 ** i == next(gen) def test_expo_max_value(): @@ -72,22 +73,6 @@ def test_constant(): assert 3 == next(gen) -def test_on_predicate_aws(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - - @backoff.on_predicate(backoff.aws_expo, - jitter=lambda: 0) - def return_true(log, n): - val = (len(log) == n - 1) - log.append(val) - return val - - log = [] - ret = return_true(log, 3) - assert ret is True - assert 3 == len(log) - - def test_on_predicate(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -103,21 +88,6 @@ def return_true(log, n): assert 3 == len(log) -def test_on_predicate_aws_max_tries(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - - @backoff.on_predicate(backoff.aws_expo, max_tries=3) - def return_true(log, n): - val = (len(log) == n) - log.append(val) - return val - - log = [] - ret = return_true(log, 10) - assert ret is False - assert 3 == len(log) - - def test_on_predicate_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -133,22 +103,6 @@ def return_true(log, n): assert 3 == len(log) -def test_on_exception_aws(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - - @backoff.on_exception(backoff.aws_expo, KeyError) - def keyerror_then_true(log, n): - if len(log) == n: - return True - e = KeyError() - log.append(e) - raise e - - log = [] - assert keyerror_then_true(log, 3) is True - assert 3 == len(log) - - def test_on_exception(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -165,27 +119,6 @@ def keyerror_then_true(log, n): assert 3 == len(log) -def test_on_exception_aws_tuple(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - - @backoff.on_exception(backoff.aws_expo, (KeyError, ValueError)) - def keyerror_valueerror_then_true(log): - if len(log) == 2: - return True - if len(log) == 0: - e = KeyError() - if len(log) == 1: - e = ValueError() - log.append(e) - raise e - - log = [] - assert keyerror_valueerror_then_true(log) is True - assert 2 == len(log) - assert isinstance(log[0], KeyError) - assert isinstance(log[1], ValueError) - - def test_on_exception_tuple(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -207,24 +140,6 @@ def keyerror_valueerror_then_true(log): assert isinstance(log[1], ValueError) -def test_on_exception_aws_max_tries(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - - @backoff.on_exception(backoff.aws_expo, KeyError, max_tries=3) - def keyerror_then_true(log, n, foo=None): - if len(log) == n: - return True - e = KeyError() - log.append(e) - raise e - - log = [] - with pytest.raises(KeyError): - keyerror_then_true(log, 10, foo="bar") - - assert 3 == len(log) - - def test_on_exception_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) @@ -280,7 +195,101 @@ def _save_target(f): return f -def test_on_exception_success(): +def test_on_exception_success_random_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.random_jitter, + init_value=0.5) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details['wait'] >= 0.5 * 2 ** i + + +def test_on_exception_success_full_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.full_jitter, + init_value=0.5) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details['wait'] <= 0.5 * 2 ** i + + +def test_on_exception_success_equal_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.equal_jitter, + init_value=0.5) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details['wait'] >= (0.5 * 2 ** i) / 2.0 + assert details['wait'] <= 0.5 * 2 ** i + + +def test_on_exception_success(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr('random.random', lambda: 1) + log, log_success, log_backoff, log_giveup = _log_hdlrs() @backoff.on_exception(backoff.constant, @@ -288,7 +297,7 @@ def test_on_exception_success(): on_success=log_success, on_backoff=log_backoff, on_giveup=log_giveup, - jitter=lambda: 0, + jitter=backoff.random_jitter, interval=0) @_save_target def succeeder(*args, **kwargs): @@ -309,7 +318,7 @@ def succeeder(*args, **kwargs): 'kwargs': {'foo': 1, 'bar': 2}, 'target': succeeder._target, 'tries': i + 1, - 'wait': 0} + 'wait': 1} details = log['success'][0] assert details == {'args': (1, 2, 3), @@ -327,7 +336,7 @@ def test_on_exception_giveup(): on_backoff=log_backoff, on_giveup=log_giveup, max_tries=3, - jitter=lambda: 0, + jitter=backoff.random_jitter, interval=0) @_save_target def exceptor(*args, **kwargs): @@ -355,7 +364,7 @@ def test_on_predicate_success(): on_success=log_success, on_backoff=log_backoff, on_giveup=log_giveup, - jitter=lambda: 0, + jitter=backoff.full_jitter, interval=0) @_save_target def success(*args, **kwargs): @@ -394,7 +403,6 @@ def test_on_predicate_giveup(): on_backoff=log_backoff, on_giveup=log_giveup, max_tries=3, - jitter=lambda: 0, interval=0) @_save_target def emptiness(*args, **kwargs): @@ -423,7 +431,7 @@ def test_on_predicate_iterable_handlers(): on_backoff=(h[2] for h in hdlrs), on_giveup=(h[3] for h in hdlrs), max_tries=3, - jitter=lambda: 0, + jitter=backoff.equal_jitter, interval=0) @_save_target def emptiness(*args, **kwargs): From 323a746f53ee21f94002f8c3dbbdd73861a0e010 Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Wed, 20 Jan 2016 11:45:21 +0800 Subject: [PATCH 03/11] refactor code to support random, full, and equal jitter --- README.md | 16 ++++++---------- backoff.py | 4 ++-- backoff_tests.py | 4 ++-- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index a77667a..fb88271 100644 --- a/README.md +++ b/README.md @@ -148,26 +148,22 @@ set the logger level to INFO: logging.getLogger('backoff').setLevel(logging.INFO) -## Examples for Full Jitter +## Examples for Full Jitter and Equal Jitter -*Full Jitter algorithm comes from [AWS Blog](http://www.awsarchitectureblog.com/2015/03/backoff.html), basically usage is almost identical to aforementioned examples, hence the unique difference would be described in this session.* +*Full Jitter and Equal Jitter algorithm comes from [AWS Blog](http://www.awsarchitectureblog.com/2015/03/backoff.html), basically usage is almost identical to aforementioned examples, hence the unique difference would be described in this session.* ### @backoff.on_exception The ``on_exception`` decorator is used to retry when a specified exception is raised. Here's an example using AWS exponential backoff when any requests exception is raised: - @backoff.on_exception(backoff.aws_expo, - jitter=lambda: 0, + @backoff.on_exception(backoff.expo, + jitter=backoff.full_jitter, requests.exceptions.RequestException, max_tries=8) def get_url(url): return requests.get(url) -To take advantage of Full Jitter, you may just specify ``backoff.aws_expo`` and it should work as expected - - sleep = random_between(0, min(max_value, base * 2 ** attempt)) - -Actually both ``on_exception`` and ``on_predicate`` does slightly jitter as well by default, the idea is adding additional random milliseconds. Since ``aws_expo`` includes jitter, you are able to disable default random jitter by specifying ``jitter=lambda: 0``. +To take advantage of Full Jitter, you may just specify ``backoff.expo``, ``jitter=backoff.full_jitter`` and it should work as expected. ## Make sure you are ready to 'commit' @@ -212,7 +208,7 @@ Actually both ``on_exception`` and ``on_predicate`` does slightly jitter as well --------------- coverage: platform darwin, python 2.7.10-final-0 --------------- Name Stmts Miss Cover Missing ------------------------------------------ - backoff.py 123 0 100% + backoff.py 125 0 100% ========================== 25 passed in 0.18 seconds =========================== diff --git a/backoff.py b/backoff.py index 2e2fecc..b43836d 100644 --- a/backoff.py +++ b/backoff.py @@ -233,7 +233,7 @@ def equal_jitter(value): def on_predicate(wait_gen, predicate=operator.not_, max_tries=None, - jitter=None, + jitter=random_jitter, on_success=None, on_backoff=None, on_giveup=None, @@ -327,7 +327,7 @@ def retry(*args, **kwargs): def on_exception(wait_gen, exception, max_tries=None, - jitter=None, + jitter=random_jitter, on_success=None, on_backoff=None, on_giveup=None, diff --git a/backoff_tests.py b/backoff_tests.py index 58f7a35..5d6d6a4 100644 --- a/backoff_tests.py +++ b/backoff_tests.py @@ -91,7 +91,7 @@ def return_true(log, n): def test_on_predicate_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) - @backoff.on_predicate(backoff.expo, max_tries=3) + @backoff.on_predicate(backoff.expo, jitter=None, max_tries=3) def return_true(log, n): val = (len(log) == n) log.append(val) @@ -143,7 +143,7 @@ def keyerror_valueerror_then_true(log): def test_on_exception_max_tries(monkeypatch): monkeypatch.setattr('time.sleep', lambda x: None) - @backoff.on_exception(backoff.expo, KeyError, max_tries=3) + @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=3) def keyerror_then_true(log, n, foo=None): if len(log) == n: return True From e5db6e908080ce8c847e7142401d0ea484b9fa55 Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Wed, 20 Jan 2016 14:44:18 +0800 Subject: [PATCH 04/11] refactor code to support random, full, and equal jitter --- README.md | 2 +- backoff.py | 30 +++++++++++------ backoff_tests.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fb88271..d2cca7f 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ To take advantage of Full Jitter, you may just specify ``backoff.expo``, ``jitte --------------- coverage: platform darwin, python 2.7.10-final-0 --------------- Name Stmts Miss Cover Missing ------------------------------------------ - backoff.py 125 0 100% + backoff.py 131 0 100% ========================== 25 passed in 0.18 seconds =========================== diff --git a/backoff.py b/backoff.py index b43836d..edfdb4d 100644 --- a/backoff.py +++ b/backoff.py @@ -233,7 +233,7 @@ def equal_jitter(value): def on_predicate(wait_gen, predicate=operator.not_, max_tries=None, - jitter=random_jitter, + jitter=full_jitter, on_success=None, on_backoff=None, on_giveup=None, @@ -292,10 +292,15 @@ def retry(*args, **kwargs): 'value': ret}) break - if jitter is not None: - seconds = jitter(next(wait)) - else: - seconds = next(wait) + try: + if jitter is not None: + seconds = jitter(next(wait)) + else: + seconds = next(wait) + except TypeError: + # support deprecated nullary jitter function signature + # which returns a delta rather than a jittered value + seconds = next(wait) + jitter() for hdlr in backoff_hdlrs: hdlr({'target': target, @@ -327,7 +332,7 @@ def retry(*args, **kwargs): def on_exception(wait_gen, exception, max_tries=None, - jitter=random_jitter, + jitter=full_jitter, on_success=None, on_backoff=None, on_giveup=None, @@ -384,10 +389,15 @@ def retry(*args, **kwargs): 'tries': tries}) raise - if jitter is not None: - seconds = jitter(next(wait)) - else: - seconds = next(wait) + try: + if jitter is not None: + seconds = jitter(next(wait)) + else: + seconds = next(wait) + except TypeError: + # support deprecated nullary jitter function signature + # which returns a delta rather than a jittered value + seconds = next(wait) + jitter() for hdlr in backoff_hdlrs: hdlr({'target': target, diff --git a/backoff_tests.py b/backoff_tests.py index 5d6d6a4..06a0c6a 100644 --- a/backoff_tests.py +++ b/backoff_tests.py @@ -4,6 +4,7 @@ import collections import functools import pytest +import random def test_full_jitter(): @@ -450,3 +451,89 @@ def emptiness(*args, **kwargs): 'target': emptiness._target, 'tries': 3, 'value': None} + + +# To maintain backward compatibility, +# on_predicate should support 0-argument jitter function. +def test_on_exception_success_0_arg_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr('random.random', lambda: 0) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_exception(backoff.constant, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=random.random, + interval=0) + @_save_target + def succeeder(*args, **kwargs): + # succeed after we've backed off twice + if len(log['backoff']) < 2: + raise ValueError("catch me") + + succeeder(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': succeeder._target, + 'tries': i + 1, + 'wait': 0} + + details = log['success'][0] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': succeeder._target, + 'tries': 3} + + +# To maintain backward compatibility, +# on_predicate should support 0-argument jitter function. +def test_on_predicate_success_0_arg_jitter(monkeypatch): + monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr('random.random', lambda: 0) + + log, log_success, log_backoff, log_giveup = _log_hdlrs() + + @backoff.on_predicate(backoff.constant, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=random.random, + interval=0) + @_save_target + def success(*args, **kwargs): + # succeed after we've backed off twice + return len(log['backoff']) == 2 + + success(1, 2, 3, foo=1, bar=2) + + # we try 3 times, backing off twice before succeeding + assert len(log['success']) == 1 + assert len(log['backoff']) == 2 + assert len(log['giveup']) == 0 + + for i in range(2): + details = log['backoff'][i] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': success._target, + 'tries': i + 1, + 'value': False, + 'wait': 0} + + details = log['success'][0] + assert details == {'args': (1, 2, 3), + 'kwargs': {'foo': 1, 'bar': 2}, + 'target': success._target, + 'tries': 3, + 'value': True} From 005b516806f3ffd2b852404a7373460534e3a33b Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Wed, 20 Jan 2016 20:53:44 +0800 Subject: [PATCH 05/11] Test Travis CI --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d2cca7f..07adec8 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![Build Status](https://travis-ci.org/litl/backoff.svg?branch=master)](https://travis-ci.org/litl/backoff?branch=master) [![Coverage Status](https://coveralls.io/repos/litl/backoff/badge.svg?branch=master)](https://coveralls.io/r/litl/backoff?branch=master) +https://api.travis-ci.org/trendmicro/backoff-python.svg?branch=master + Function decoration for backoff and retry This module provides function decorators which can be used to wrap a From ddaaa14fa04a43fa9f599ce4253bdd12e52fc5a6 Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Wed, 20 Jan 2016 20:57:31 +0800 Subject: [PATCH 06/11] Test Travis CI --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 07adec8..d2cca7f 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ [![Build Status](https://travis-ci.org/litl/backoff.svg?branch=master)](https://travis-ci.org/litl/backoff?branch=master) [![Coverage Status](https://coveralls.io/repos/litl/backoff/badge.svg?branch=master)](https://coveralls.io/r/litl/backoff?branch=master) -https://api.travis-ci.org/trendmicro/backoff-python.svg?branch=master - Function decoration for backoff and retry This module provides function decorators which can be used to wrap a From 6818d4abb3453af932de7493a4896e06af33415e Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Wed, 20 Jan 2016 21:04:49 +0800 Subject: [PATCH 07/11] Test Coveralls --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2cca7f..8b5a76b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # backoff [![Build Status](https://travis-ci.org/litl/backoff.svg?branch=master)](https://travis-ci.org/litl/backoff?branch=master) [![Coverage Status](https://coveralls.io/repos/litl/backoff/badge.svg?branch=master)](https://coveralls.io/r/litl/backoff?branch=master) - + Function decoration for backoff and retry This module provides function decorators which can be used to wrap a From 8000fda0230224ec5031598dc314017b8126153d Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Wed, 20 Jan 2016 21:11:44 +0800 Subject: [PATCH 08/11] Test Coveralls --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b5a76b..d2cca7f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # backoff [![Build Status](https://travis-ci.org/litl/backoff.svg?branch=master)](https://travis-ci.org/litl/backoff?branch=master) [![Coverage Status](https://coveralls.io/repos/litl/backoff/badge.svg?branch=master)](https://coveralls.io/r/litl/backoff?branch=master) - + Function decoration for backoff and retry This module provides function decorators which can be used to wrap a From d70d48004e37e6a0807f19fa25127adbc3ba6286 Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Thu, 21 Jan 2016 10:42:53 +0800 Subject: [PATCH 09/11] rollback README.md --- README.md | 64 ------------------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/README.md b/README.md index d2cca7f..37c539e 100644 --- a/README.md +++ b/README.md @@ -148,67 +148,3 @@ set the logger level to INFO: logging.getLogger('backoff').setLevel(logging.INFO) -## Examples for Full Jitter and Equal Jitter - -*Full Jitter and Equal Jitter algorithm comes from [AWS Blog](http://www.awsarchitectureblog.com/2015/03/backoff.html), basically usage is almost identical to aforementioned examples, hence the unique difference would be described in this session.* - -### @backoff.on_exception - -The ``on_exception`` decorator is used to retry when a specified exception is raised. Here's an example using AWS exponential backoff when any requests exception is raised: - - @backoff.on_exception(backoff.expo, - jitter=backoff.full_jitter, - requests.exceptions.RequestException, - max_tries=8) - def get_url(url): - return requests.get(url) - -To take advantage of Full Jitter, you may just specify ``backoff.expo``, ``jitter=backoff.full_jitter`` and it should work as expected. - -## Make sure you are ready to 'commit' - -### Virtual Python Environment builder - -``virtualenv`` is a tool to create isolated testing environments which could prevent from pollution. - - # Install virtualenv - $ sudo pip install virtualenv - - # Create virtual environment - $ virtualenv mytest - - # Activate virtual environment - $ cd mytest - $ source bin/activate - -### Python style guide checker - - (mytest)$ sudo pip install pep8 - -### Passive checker of Python programs - - (mytest)$ sudo pip install pyflakes - -### Pytest plugin for measuring coverage - - (mytest)$ sudo pip install pytest-cov - -### Secure quality for your changes - - # Switch to backoff.py and backoff_tests.py folder - (mytest)$ make check - - ============================= test session starts ============================== - platform darwin -- Python 2.7.10, pytest-2.8.5, py-1.4.31, pluggy-0.3.1 - rootdir: /Github/backoff, inifile: - plugins: cov-2.2.0 - collected 16 items - - backoff_tests.py ................ - --------------- coverage: platform darwin, python 2.7.10-final-0 --------------- - Name Stmts Miss Cover Missing - ------------------------------------------ - backoff.py 131 0 100% - - ========================== 25 passed in 0.18 seconds =========================== - From a976052b65a2a0aac00f2efbb704bf208b80781a Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Thu, 21 Jan 2016 10:44:01 +0800 Subject: [PATCH 10/11] Move next(wait) outside the try block here. The reason is in the case that the wait generator (which can be custom) throws a TypeError for some reason, the error might be masked. --- backoff.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backoff.py b/backoff.py index edfdb4d..280b0ad 100644 --- a/backoff.py +++ b/backoff.py @@ -292,15 +292,16 @@ def retry(*args, **kwargs): 'value': ret}) break + value = next(wait) try: if jitter is not None: - seconds = jitter(next(wait)) + seconds = jitter(value) else: - seconds = next(wait) + seconds = value except TypeError: # support deprecated nullary jitter function signature # which returns a delta rather than a jittered value - seconds = next(wait) + jitter() + seconds = value + jitter() for hdlr in backoff_hdlrs: hdlr({'target': target, @@ -389,15 +390,16 @@ def retry(*args, **kwargs): 'tries': tries}) raise + value = next(wait) try: if jitter is not None: - seconds = jitter(next(wait)) + seconds = jitter(value) else: - seconds = next(wait) + seconds = value except TypeError: # support deprecated nullary jitter function signature # which returns a delta rather than a jittered value - seconds = next(wait) + jitter() + seconds = value + jitter() for hdlr in backoff_hdlrs: hdlr({'target': target, From 88c61e5d9497e4a92b63aad3c1e8fd16be71b227 Mon Sep 17 00:00:00 2001 From: Jonas Cheng Date: Sat, 23 Jan 2016 11:41:27 +0800 Subject: [PATCH 11/11] revert original "jitter=lambda: 0" --- backoff_tests.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/backoff_tests.py b/backoff_tests.py index 06a0c6a..e8b546e 100644 --- a/backoff_tests.py +++ b/backoff_tests.py @@ -287,10 +287,7 @@ def succeeder(*args, **kwargs): assert details['wait'] <= 0.5 * 2 ** i -def test_on_exception_success(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - monkeypatch.setattr('random.random', lambda: 1) - +def test_on_exception_success(): log, log_success, log_backoff, log_giveup = _log_hdlrs() @backoff.on_exception(backoff.constant, @@ -298,7 +295,7 @@ def test_on_exception_success(monkeypatch): on_success=log_success, on_backoff=log_backoff, on_giveup=log_giveup, - jitter=backoff.random_jitter, + jitter=lambda: 0, interval=0) @_save_target def succeeder(*args, **kwargs): @@ -319,7 +316,7 @@ def succeeder(*args, **kwargs): 'kwargs': {'foo': 1, 'bar': 2}, 'target': succeeder._target, 'tries': i + 1, - 'wait': 1} + 'wait': 0} details = log['success'][0] assert details == {'args': (1, 2, 3), @@ -337,7 +334,7 @@ def test_on_exception_giveup(): on_backoff=log_backoff, on_giveup=log_giveup, max_tries=3, - jitter=backoff.random_jitter, + jitter=lambda: 0, interval=0) @_save_target def exceptor(*args, **kwargs): @@ -365,7 +362,7 @@ def test_on_predicate_success(): on_success=log_success, on_backoff=log_backoff, on_giveup=log_giveup, - jitter=backoff.full_jitter, + jitter=lambda: 0, interval=0) @_save_target def success(*args, **kwargs): @@ -404,6 +401,7 @@ def test_on_predicate_giveup(): on_backoff=log_backoff, on_giveup=log_giveup, max_tries=3, + jitter=lambda: 0, interval=0) @_save_target def emptiness(*args, **kwargs): @@ -432,7 +430,7 @@ def test_on_predicate_iterable_handlers(): on_backoff=(h[2] for h in hdlrs), on_giveup=(h[3] for h in hdlrs), max_tries=3, - jitter=backoff.equal_jitter, + jitter=lambda: 0, interval=0) @_save_target def emptiness(*args, **kwargs):