From 9b043c983473eeb3a0b508c18426412dc67a0a58 Mon Sep 17 00:00:00 2001 From: Ahmet Altay Date: Wed, 6 Nov 2019 11:11:50 -0800 Subject: [PATCH 1/3] Increase overhaed budget for test_sampler_transition_overhead This test is flaky when it is run under load. Increasing overhead budget as a temporary fix. A better solution would be to change the test structure to not depend on specific timings. --- sdks/python/apache_beam/runners/worker/statesampler_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sdks/python/apache_beam/runners/worker/statesampler_test.py b/sdks/python/apache_beam/runners/worker/statesampler_test.py index 176f6e52b407..b522294ead36 100644 --- a/sdks/python/apache_beam/runners/worker/statesampler_test.py +++ b/sdks/python/apache_beam/runners/worker/statesampler_test.py @@ -117,6 +117,11 @@ def test_sampler_transition_overhead(self): elapsed_time = time.time() - start_time state_transition_count = sampler.get_info().transition_count overhead_us = 1000000.0 * elapsed_time / state_transition_count + + # TODO: This test is flaky when it is run under load. A better solution + # would be to change the test structure to not depend on specific timings. + overhead_us = 2 * overhead_us + logging.info('Overhead per transition: %fus', overhead_us) # Conservative upper bound on overhead in microseconds (we expect this to # take 0.17us when compiled in opt mode or 0.48 us when compiled with in From 5688a3124466be2563ffde6752944e1e759bf8c1 Mon Sep 17 00:00:00 2001 From: Ahmet Altay Date: Wed, 6 Nov 2019 14:10:52 -0800 Subject: [PATCH 2/3] Add tenacity retry decorator --- sdks/python/apache_beam/runners/worker/statesampler_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/worker/statesampler_test.py b/sdks/python/apache_beam/runners/worker/statesampler_test.py index b522294ead36..616ccc71a702 100644 --- a/sdks/python/apache_beam/runners/worker/statesampler_test.py +++ b/sdks/python/apache_beam/runners/worker/statesampler_test.py @@ -95,6 +95,7 @@ def test_basic_sampler(self): self.assertGreater(actual_value, expected_value * (1.0 - margin_of_error)) self.assertLess(actual_value, expected_value * (1.0 + margin_of_error)) + @retry(reraise=True, stop=stop_after_attempt(3)) def test_sampler_transition_overhead(self): # Set up state sampler. counter_factory = CounterFactory() @@ -117,11 +118,11 @@ def test_sampler_transition_overhead(self): elapsed_time = time.time() - start_time state_transition_count = sampler.get_info().transition_count overhead_us = 1000000.0 * elapsed_time / state_transition_count - + # TODO: This test is flaky when it is run under load. A better solution # would be to change the test structure to not depend on specific timings. overhead_us = 2 * overhead_us - + logging.info('Overhead per transition: %fus', overhead_us) # Conservative upper bound on overhead in microseconds (we expect this to # take 0.17us when compiled in opt mode or 0.48 us when compiled with in From e5ccedc8b15769ebde7572eeb53ed6119587fb61 Mon Sep 17 00:00:00 2001 From: Ahmet Altay Date: Wed, 6 Nov 2019 15:15:13 -0800 Subject: [PATCH 3/3] fixup --- sdks/python/apache_beam/runners/worker/statesampler_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/apache_beam/runners/worker/statesampler_test.py b/sdks/python/apache_beam/runners/worker/statesampler_test.py index 616ccc71a702..97fe6d97c0cf 100644 --- a/sdks/python/apache_beam/runners/worker/statesampler_test.py +++ b/sdks/python/apache_beam/runners/worker/statesampler_test.py @@ -95,6 +95,8 @@ def test_basic_sampler(self): self.assertGreater(actual_value, expected_value * (1.0 - margin_of_error)) self.assertLess(actual_value, expected_value * (1.0 + margin_of_error)) + # TODO: This test is flaky when it is run under load. A better solution + # would be to change the test structure to not depend on specific timings. @retry(reraise=True, stop=stop_after_attempt(3)) def test_sampler_transition_overhead(self): # Set up state sampler.