From 35154c3eb61da36744c064223a0fd5313e8db858 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Tue, 31 Mar 2026 15:24:08 -0700 Subject: [PATCH 1/2] fix(tests): poll for logs in smoke tests to handle CloudWatch ingestion latency --- tests/smoketests/sdk/test_async_devbox.py | 42 +++++++--------------- tests/smoketests/sdk/test_devbox.py | 43 +++++++---------------- uv.lock | 2 +- 3 files changed, 27 insertions(+), 60 deletions(-) diff --git a/tests/smoketests/sdk/test_async_devbox.py b/tests/smoketests/sdk/test_async_devbox.py index 736d6eec8..662156c15 100644 --- a/tests/smoketests/sdk/test_async_devbox.py +++ b/tests/smoketests/sdk/test_async_devbox.py @@ -11,6 +11,7 @@ from runloop_api_client.sdk import AsyncDevbox, AsyncRunloopSDK from tests.smoketests.utils import unique_name from runloop_api_client.lib.polling import PollingConfig +from runloop_api_client.lib.polling_async import async_poll_until pytestmark = [pytest.mark.smoketest, pytest.mark.asyncio] @@ -1040,21 +1041,6 @@ async def test_shell_exec_async_with_both_streams(self, devbox: AsyncDevbox) -> class TestAsyncDevboxLogs: """Test async devbox logs retrieval functionality.""" - @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) - async def test_logs_basic(self, shared_devbox: AsyncDevbox) -> None: - """Test retrieving devbox logs returns valid response structure.""" - test_message = "async basic log test message" - result = await shared_devbox.cmd.exec(f'echo "{test_message}"') - assert result.exit_code == 0 - - logs = await shared_devbox.logs() - - assert logs is not None - assert hasattr(logs, "logs") - assert isinstance(logs.logs, list) - log_content = " ".join(str(log) for log in logs.logs) - assert test_message in log_content - @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) async def test_logs_with_execution_filter(self, shared_devbox: AsyncDevbox) -> None: """Test retrieving devbox logs filtered by execution ID.""" @@ -1062,13 +1048,12 @@ async def test_logs_with_execution_filter(self, shared_devbox: AsyncDevbox) -> N result = await shared_devbox.cmd.exec(f'echo "{test_message}"') assert result.exit_code == 0 - logs = await shared_devbox.logs(execution_id=result.execution_id) - - assert logs is not None - assert hasattr(logs, "logs") - assert isinstance(logs.logs, list) - log_content = " ".join(str(log) for log in logs.logs) - assert test_message in log_content + logs = await async_poll_until( + retriever=lambda: shared_devbox.logs(execution_id=result.execution_id), + is_terminal=lambda l: any(test_message in (log.message or "") for log in l.logs), + config=PollingConfig(timeout_seconds=10, interval_seconds=1), + ) + assert any(test_message in (log.message or "") for log in logs.logs) @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) async def test_logs_with_shell_name_filter(self, shared_devbox: AsyncDevbox) -> None: @@ -1080,10 +1065,9 @@ async def test_logs_with_shell_name_filter(self, shared_devbox: AsyncDevbox) -> result = await shell.exec(f'echo "{test_message}"') assert result.exit_code == 0 - logs = await shared_devbox.logs(shell_name=shell_name) - - assert logs is not None - assert hasattr(logs, "logs") - assert isinstance(logs.logs, list) - log_content = " ".join(str(log) for log in logs.logs) - assert test_message in log_content + logs = await async_poll_until( + retriever=lambda: shared_devbox.logs(shell_name=shell_name), + is_terminal=lambda l: any(test_message in (log.message or "") for log in l.logs), + config=PollingConfig(timeout_seconds=10, interval_seconds=1), + ) + assert any(test_message in (log.message or "") for log in logs.logs) diff --git a/tests/smoketests/sdk/test_devbox.py b/tests/smoketests/sdk/test_devbox.py index 88b2bad2d..c516b530b 100644 --- a/tests/smoketests/sdk/test_devbox.py +++ b/tests/smoketests/sdk/test_devbox.py @@ -10,7 +10,7 @@ from runloop_api_client.sdk import Devbox, RunloopSDK from tests.smoketests.utils import unique_name -from runloop_api_client.lib.polling import PollingConfig +from runloop_api_client.lib.polling import PollingConfig, poll_until pytestmark = [pytest.mark.smoketest] @@ -1026,21 +1026,6 @@ def test_shell_exec_async_with_both_streams(self, devbox: Devbox) -> None: class TestDevboxLogs: """Test devbox logs retrieval functionality.""" - @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) - def test_logs_basic(self, shared_devbox: Devbox) -> None: - """Test retrieving devbox logs returns valid response structure.""" - test_message = "basic log test message" - result = shared_devbox.cmd.exec(f'echo "{test_message}"') - assert result.exit_code == 0 - - logs = shared_devbox.logs() - - assert logs is not None - assert hasattr(logs, "logs") - assert isinstance(logs.logs, list) - log_content = " ".join(str(log) for log in logs.logs) - assert test_message in log_content - @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) def test_logs_with_execution_filter(self, shared_devbox: Devbox) -> None: """Test retrieving devbox logs filtered by execution ID.""" @@ -1048,13 +1033,12 @@ def test_logs_with_execution_filter(self, shared_devbox: Devbox) -> None: result = shared_devbox.cmd.exec(f'echo "{test_message}"') assert result.exit_code == 0 - logs = shared_devbox.logs(execution_id=result.execution_id) - - assert logs is not None - assert hasattr(logs, "logs") - assert isinstance(logs.logs, list) - log_content = " ".join(str(log) for log in logs.logs) - assert test_message in log_content + logs = poll_until( + retriever=lambda: shared_devbox.logs(execution_id=result.execution_id), + is_terminal=lambda l: any(test_message in (log.message or "") for log in l.logs), + config=PollingConfig(timeout_seconds=10, interval_seconds=1), + ) + assert any(test_message in (log.message or "") for log in logs.logs) @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) def test_logs_with_shell_name_filter(self, shared_devbox: Devbox) -> None: @@ -1066,10 +1050,9 @@ def test_logs_with_shell_name_filter(self, shared_devbox: Devbox) -> None: result = shell.exec(f'echo "{test_message}"') assert result.exit_code == 0 - logs = shared_devbox.logs(shell_name=shell_name) - - assert logs is not None - assert hasattr(logs, "logs") - assert isinstance(logs.logs, list) - log_content = " ".join(str(log) for log in logs.logs) - assert test_message in log_content + logs = poll_until( + retriever=lambda: shared_devbox.logs(shell_name=shell_name), + is_terminal=lambda l: any(test_message in (log.message or "") for log in l.logs), + config=PollingConfig(timeout_seconds=10, interval_seconds=1), + ) + assert any(test_message in (log.message or "") for log in logs.logs) diff --git a/uv.lock b/uv.lock index bac2c9a14..536ecad8e 100644 --- a/uv.lock +++ b/uv.lock @@ -2386,7 +2386,7 @@ wheels = [ [[package]] name = "runloop-api-client" -version = "1.13.2" +version = "1.13.3" source = { editable = "." } dependencies = [ { name = "anyio" }, From 43a9eb074dd49e32175e73f7c8cd92baf2032f24 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Tue, 31 Mar 2026 16:46:00 -0700 Subject: [PATCH 2/2] address comments --- tests/smoketests/sdk/test_async_devbox.py | 17 +++++++++++++++++ tests/smoketests/sdk/test_devbox.py | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/tests/smoketests/sdk/test_async_devbox.py b/tests/smoketests/sdk/test_async_devbox.py index 662156c15..68af2b508 100644 --- a/tests/smoketests/sdk/test_async_devbox.py +++ b/tests/smoketests/sdk/test_async_devbox.py @@ -1041,6 +1041,23 @@ async def test_shell_exec_async_with_both_streams(self, devbox: AsyncDevbox) -> class TestAsyncDevboxLogs: """Test async devbox logs retrieval functionality.""" + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) + async def test_logs_basic(self, shared_devbox: AsyncDevbox) -> None: + """Test retrieving unfiltered devbox logs.""" + test_message = "async basic log test message" + result = await shared_devbox.cmd.exec(f'echo "{test_message}"') + assert result.exit_code == 0 + + # Log ingestion is async — logs may not be queryable immediately after + # command execution. Poll every 1s with a 10s timeout (well within the + # 30s test timeout) to accommodate variable ingestion latency. + logs = await async_poll_until( + retriever=lambda: shared_devbox.logs(), + is_terminal=lambda l: any(test_message in (log.message or "") for log in l.logs), + config=PollingConfig(timeout_seconds=10, interval_seconds=1), + ) + assert any(test_message in (log.message or "") for log in logs.logs) + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) async def test_logs_with_execution_filter(self, shared_devbox: AsyncDevbox) -> None: """Test retrieving devbox logs filtered by execution ID.""" diff --git a/tests/smoketests/sdk/test_devbox.py b/tests/smoketests/sdk/test_devbox.py index c516b530b..cedd1bc75 100644 --- a/tests/smoketests/sdk/test_devbox.py +++ b/tests/smoketests/sdk/test_devbox.py @@ -1026,6 +1026,23 @@ def test_shell_exec_async_with_both_streams(self, devbox: Devbox) -> None: class TestDevboxLogs: """Test devbox logs retrieval functionality.""" + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) + def test_logs_basic(self, shared_devbox: Devbox) -> None: + """Test retrieving unfiltered devbox logs.""" + test_message = "basic log test message" + result = shared_devbox.cmd.exec(f'echo "{test_message}"') + assert result.exit_code == 0 + + # Log ingestion is async — logs may not be queryable immediately after + # command execution. Poll every 1s with a 10s timeout (well within the + # 30s test timeout) to accommodate variable ingestion latency. + logs = poll_until( + retriever=lambda: shared_devbox.logs(), + is_terminal=lambda l: any(test_message in (log.message or "") for log in l.logs), + config=PollingConfig(timeout_seconds=10, interval_seconds=1), + ) + assert any(test_message in (log.message or "") for log in logs.logs) + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) def test_logs_with_execution_filter(self, shared_devbox: Devbox) -> None: """Test retrieving devbox logs filtered by execution ID."""