From 40a5f005c08a1eeac628b38493a861e028da200d Mon Sep 17 00:00:00 2001 From: crusaderky Date: Thu, 9 Jun 2022 20:49:10 +0100 Subject: [PATCH 1/2] Fix flaky test_gather_dep_one_worker_always_busy --- distributed/tests/test_worker.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py index e4c1cab17d4..9d5b54ed799 100644 --- a/distributed/tests/test_worker.py +++ b/distributed/tests/test_worker.py @@ -1939,6 +1939,7 @@ def f(x): assert not C.instances +@pytest.mark.slow @gen_cluster(client=True) async def test_gather_dep_one_worker_always_busy(c, s, a, b): # Ensure that both dependencies for H are on another worker than H itself. @@ -1966,12 +1967,14 @@ async def test_gather_dep_one_worker_always_busy(c, s, a, b): assert b.tasks[g.key].state in ("flight", "fetch") with pytest.raises(asyncio.TimeoutError): - await h.result(timeout=0.5) + await h.result(timeout=0.8) story = b.story("busy-gather") - # 1 busy response straight away, followed by 1 retry every 150ms for 500ms. + # 1 busy response straight away, followed by 1 retry every 150ms for 800ms. # The requests for b and g are clustered together in single messages. - assert 3 <= len(story) <= 7 + # We need to be very lax in measuring as PeriodicCallback has been observed on CI to + # occasionally lag behind by >350ms. + assert 2 <= len(story) <= 8 async with Worker(s.address, name="x") as x: # We "scatter" the data to another worker which is able to serve this data. From 07860b345d0c4de85a5cf54b6c4e470ff1b6c32f Mon Sep 17 00:00:00 2001 From: crusaderky Date: Thu, 9 Jun 2022 20:53:39 +0100 Subject: [PATCH 2/2] tweak --- distributed/tests/test_worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py index 9d5b54ed799..bbf03fa68ab 100644 --- a/distributed/tests/test_worker.py +++ b/distributed/tests/test_worker.py @@ -1972,8 +1972,8 @@ async def test_gather_dep_one_worker_always_busy(c, s, a, b): story = b.story("busy-gather") # 1 busy response straight away, followed by 1 retry every 150ms for 800ms. # The requests for b and g are clustered together in single messages. - # We need to be very lax in measuring as PeriodicCallback has been observed on CI to - # occasionally lag behind by >350ms. + # We need to be very lax in measuring as PeriodicCallback+network comms have been + # observed on CI to occasionally lag behind by several hundreds of ms. assert 2 <= len(story) <= 8 async with Worker(s.address, name="x") as x: