From 5e37a2e62694462f4a8535ed8070fb10909349b5 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 24 Nov 2025 20:24:07 -0800 Subject: [PATCH 01/13] added scorer class (kept create and list as static methods for now since we don't know how we're creating scorers yet) --- src/runloop_api_client/sdk/__init__.py | 4 + src/runloop_api_client/sdk/_types.py | 17 +++ src/runloop_api_client/sdk/async_scorer.py | 130 +++++++++++++++++++++ src/runloop_api_client/sdk/scorer.py | 130 +++++++++++++++++++++ 4 files changed, 281 insertions(+) create mode 100644 src/runloop_api_client/sdk/async_scorer.py create mode 100644 src/runloop_api_client/sdk/scorer.py diff --git a/src/runloop_api_client/sdk/__init__.py b/src/runloop_api_client/sdk/__init__.py index b08b5bf87..963215bbf 100644 --- a/src/runloop_api_client/sdk/__init__.py +++ b/src/runloop_api_client/sdk/__init__.py @@ -14,10 +14,12 @@ AsyncStorageObjectOps, ) from .devbox import Devbox, NamedShell +from .scorer import Scorer from .snapshot import Snapshot from .blueprint import Blueprint from .execution import Execution from .async_devbox import AsyncDevbox, AsyncNamedShell +from .async_scorer import AsyncScorer from .async_snapshot import AsyncSnapshot from .storage_object import StorageObject from .async_blueprint import AsyncBlueprint @@ -48,6 +50,8 @@ "AsyncExecutionResult", "Blueprint", "AsyncBlueprint", + "Scorer", + "AsyncScorer", "Snapshot", "AsyncSnapshot", "StorageObject", diff --git a/src/runloop_api_client/sdk/_types.py b/src/runloop_api_client/sdk/_types.py index 9cb0e21a9..028cb1805 100644 --- a/src/runloop_api_client/sdk/_types.py +++ b/src/runloop_api_client/sdk/_types.py @@ -4,6 +4,7 @@ from .._types import Body, Query, Headers, Timeout, NotGiven from ..lib.polling import PollingConfig from ..types.devboxes import DiskSnapshotListParams, DiskSnapshotUpdateParams +from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams, ScorerValidateParams from ..types.devbox_list_params import DevboxListParams from ..types.object_list_params import ObjectListParams from ..types.devbox_create_params import DevboxCreateParams, DevboxBaseCreateParams @@ -140,3 +141,19 @@ class SDKObjectCreateParams(ObjectCreateParams, LongRequestOptions): class SDKObjectDownloadParams(ObjectDownloadParams, BaseRequestOptions): pass + + +class SDKScorerCreateParams(ScorerCreateParams, LongRequestOptions): + pass + + +class SDKScorerListParams(ScorerListParams, BaseRequestOptions): + pass + + +class SDKScorerUpdateParams(ScorerUpdateParams, LongRequestOptions): + pass + + +class SDKScorerValidateParams(ScorerValidateParams, LongRequestOptions): + pass diff --git a/src/runloop_api_client/sdk/async_scorer.py b/src/runloop_api_client/sdk/async_scorer.py new file mode 100644 index 000000000..a48dc81d2 --- /dev/null +++ b/src/runloop_api_client/sdk/async_scorer.py @@ -0,0 +1,130 @@ +"""Scorer resource class for asynchronous operations.""" + +from __future__ import annotations + +from typing_extensions import Unpack, override + +from ._types import ( + BaseRequestOptions, + SDKScorerListParams, + SDKScorerCreateParams, + SDKScorerUpdateParams, + SDKScorerValidateParams, +) +from .._client import AsyncRunloop +from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse + + +class AsyncScorer: + """Asynchronous wrapper around a scenario scorer resource.""" + + def __init__( + self, + client: AsyncRunloop, + scorer_id: str, + ) -> None: + """Initialize the wrapper. + + :param client: Generated AsyncRunloop client + :type client: AsyncRunloop + :param scorer_id: Scorer ID returned by the API + :type scorer_id: str + """ + self._client = client + self._id = scorer_id + + @override + def __repr__(self) -> str: + return f"" + + @property + def id(self) -> str: + """Return the scorer ID. + + :return: Unique scorer ID + :rtype: str + """ + return self._id + + # TODO: replace static method once we have a proper client + @staticmethod + async def create( + client: AsyncRunloop, + **params: Unpack[SDKScorerCreateParams], + ) -> "AsyncScorer": + """Create a new scenario scorer. + + :param client: Generated AsyncRunloop client + :type client: AsyncRunloop + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters + :return: Wrapper bound to the newly created scorer + :rtype: AsyncScorer + """ + response = await client.scenarios.scorers.create( + **params, + ) + return AsyncScorer(client, response.id) + + # TODO: replace static method once we have a proper client + @staticmethod + async def list( + client: AsyncRunloop, + **params: Unpack[SDKScorerListParams], + ) -> list["AsyncScorer"]: + """List all scenario scorers. + + :param client: Generated AsyncRunloop client + :type client: AsyncRunloop + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters + :return: List of scorer wrappers + :rtype: list[AsyncScorer] + """ + page = await client.scenarios.scorers.list( + **params, + ) + return [AsyncScorer(client, item.id) async for item in page] + + async def get_info( + self, + **options: Unpack[BaseRequestOptions], + ) -> ScorerRetrieveResponse: + """Retrieve the latest scorer details. + + :param options: Optional request configuration + :return: API response describing the scorer + :rtype: ScorerRetrieveResponse + """ + return await self._client.scenarios.scorers.retrieve( + self._id, + **options, + ) + + async def update( + self, + **params: Unpack[SDKScorerUpdateParams], + ) -> ScorerUpdateResponse: + """Update the scorer. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerUpdateParams` for available parameters + :return: API response with updated scorer details + :rtype: ScorerUpdateResponse + """ + return await self._client.scenarios.scorers.update( + self._id, + **params, + ) + + async def validate( + self, + **params: Unpack[SDKScorerValidateParams], + ) -> ScorerValidateResponse: + """Validate the scorer with a given context. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters + :return: API response with validation results + :rtype: ScorerValidateResponse + """ + return await self._client.scenarios.scorers.validate( + self._id, + **params, + ) diff --git a/src/runloop_api_client/sdk/scorer.py b/src/runloop_api_client/sdk/scorer.py new file mode 100644 index 000000000..2140bada1 --- /dev/null +++ b/src/runloop_api_client/sdk/scorer.py @@ -0,0 +1,130 @@ +"""Scorer resource class for synchronous operations.""" + +from __future__ import annotations + +from typing_extensions import Unpack, override + +from ._types import ( + BaseRequestOptions, + SDKScorerListParams, + SDKScorerCreateParams, + SDKScorerUpdateParams, + SDKScorerValidateParams, +) +from .._client import Runloop +from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse + + +class Scorer: + """Synchronous wrapper around a scenario scorer resource.""" + + def __init__( + self, + client: Runloop, + scorer_id: str, + ) -> None: + """Initialize the wrapper. + + :param client: Generated Runloop client + :type client: Runloop + :param scorer_id: Scorer ID returned by the API + :type scorer_id: str + """ + self._client = client + self._id = scorer_id + + @override + def __repr__(self) -> str: + return f"" + + @property + def id(self) -> str: + """Return the scorer ID. + + :return: Unique scorer ID + :rtype: str + """ + return self._id + + # TODO: replace static method once we have a proper client + @staticmethod + def create( + client: Runloop, + **params: Unpack[SDKScorerCreateParams], + ) -> "Scorer": + """Create a new scenario scorer. + + :param client: Generated Runloop client + :type client: Runloop + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters + :return: Wrapper bound to the newly created scorer + :rtype: Scorer + """ + response = client.scenarios.scorers.create( + **params, + ) + return Scorer(client, response.id) + + # TODO: replace static method once we have a proper client + @staticmethod + def list( + client: Runloop, + **params: Unpack[SDKScorerListParams], + ) -> list["Scorer"]: + """List all scenario scorers. + + :param client: Generated Runloop client + :type client: Runloop + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters + :return: List of scorer wrappers + :rtype: list[Scorer] + """ + page = client.scenarios.scorers.list( + **params, + ) + return [Scorer(client, item.id) for item in page] + + def get_info( + self, + **options: Unpack[BaseRequestOptions], + ) -> ScorerRetrieveResponse: + """Retrieve the latest scorer details. + + :param options: Optional request configuration + :return: API response describing the scorer + :rtype: ScorerRetrieveResponse + """ + return self._client.scenarios.scorers.retrieve( + self._id, + **options, + ) + + def update( + self, + **params: Unpack[SDKScorerUpdateParams], + ) -> ScorerUpdateResponse: + """Update the scorer. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerUpdateParams` for available parameters + :return: API response with updated scorer details + :rtype: ScorerUpdateResponse + """ + return self._client.scenarios.scorers.update( + self._id, + **params, + ) + + def validate( + self, + **params: Unpack[SDKScorerValidateParams], + ) -> ScorerValidateResponse: + """Validate the scorer with a given context. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters + :return: API response with validation results + :rtype: ScorerValidateResponse + """ + return self._client.scenarios.scorers.validate( + self._id, + **params, + ) From 94435a7781d9dea015381b9a68c2017de2c2fd09 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 13:40:23 -0500 Subject: [PATCH 02/13] refactored static methods to ScorerOps class --- src/runloop_api_client/sdk/__init__.py | 5 +- src/runloop_api_client/sdk/async_.py | 68 ++++++++++++++++++++++ src/runloop_api_client/sdk/async_scorer.py | 40 ------------- src/runloop_api_client/sdk/scorer.py | 40 ------------- src/runloop_api_client/sdk/sync.py | 68 ++++++++++++++++++++++ 5 files changed, 140 insertions(+), 81 deletions(-) diff --git a/src/runloop_api_client/sdk/__init__.py b/src/runloop_api_client/sdk/__init__.py index 963215bbf..48b5e3103 100644 --- a/src/runloop_api_client/sdk/__init__.py +++ b/src/runloop_api_client/sdk/__init__.py @@ -5,9 +5,10 @@ from __future__ import annotations -from .sync import DevboxOps, RunloopSDK, SnapshotOps, BlueprintOps, StorageObjectOps +from .sync import DevboxOps, ScorerOps, RunloopSDK, SnapshotOps, BlueprintOps, StorageObjectOps from .async_ import ( AsyncDevboxOps, + AsyncScorerOps, AsyncRunloopSDK, AsyncSnapshotOps, AsyncBlueprintOps, @@ -37,6 +38,8 @@ "AsyncDevboxOps", "BlueprintOps", "AsyncBlueprintOps", + "ScorerOps", + "AsyncScorerOps", "SnapshotOps", "AsyncSnapshotOps", "StorageObjectOps", diff --git a/src/runloop_api_client/sdk/async_.py b/src/runloop_api_client/sdk/async_.py index 2dc4562c2..9931eeb7a 100644 --- a/src/runloop_api_client/sdk/async_.py +++ b/src/runloop_api_client/sdk/async_.py @@ -16,8 +16,10 @@ LongRequestOptions, SDKDevboxListParams, SDKObjectListParams, + SDKScorerListParams, SDKDevboxCreateParams, SDKObjectCreateParams, + SDKScorerCreateParams, SDKBlueprintListParams, SDKBlueprintCreateParams, SDKDiskSnapshotListParams, @@ -27,6 +29,7 @@ from .._client import DEFAULT_MAX_RETRIES, AsyncRunloop from ._helpers import detect_content_type from .async_devbox import AsyncDevbox +from .async_scorer import AsyncScorer from .async_snapshot import AsyncSnapshot from .async_blueprint import AsyncBlueprint from .async_storage_object import AsyncStorageObject @@ -475,6 +478,67 @@ async def upload_from_bytes( return obj +class AsyncScorerOps: + """High-level async manager for creating and managing scenario scorers. + + Accessed via ``runloop.scorer`` from :class:`AsyncRunloopSDK`, provides + coroutines to create, list, and access scorers. + + Example: + >>> runloop = AsyncRunloopSDK() + >>> scorer = await runloop.scorer.create(name="my-scorer", scorer_type="llm_judge") + >>> scorers = await runloop.scorer.list() + """ + + def __init__(self, client: AsyncRunloop) -> None: + """Initialize the manager. + + :param client: Generated AsyncRunloop client to wrap + :type client: AsyncRunloop + """ + self._client = client + + async def create( + self, + **params: Unpack[SDKScorerCreateParams], + ) -> AsyncScorer: + """Create a new scenario scorer. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters + :return: Wrapper bound to the newly created scorer + :rtype: AsyncScorer + """ + response = await self._client.scenarios.scorers.create( + **params, + ) + return AsyncScorer(self._client, response.id) + + def from_id(self, scorer_id: str) -> AsyncScorer: + """Return a scorer wrapper for the given ID. + + :param scorer_id: Scorer ID to wrap + :type scorer_id: str + :return: Wrapper for the scorer resource + :rtype: AsyncScorer + """ + return AsyncScorer(self._client, scorer_id) + + async def list( + self, + **params: Unpack[SDKScorerListParams], + ) -> list[AsyncScorer]: + """List all scenario scorers. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters + :return: List of scorer wrappers + :rtype: list[AsyncScorer] + """ + page = await self._client.scenarios.scorers.list( + **params, + ) + return [AsyncScorer(self._client, item.id) async for item in page] + + class AsyncRunloopSDK: """High-level asynchronous entry point for the Runloop SDK. @@ -488,6 +552,8 @@ class AsyncRunloopSDK: :vartype devbox: AsyncDevboxOps :ivar blueprint: High-level async interface for blueprint management :vartype blueprint: AsyncBlueprintOps + :ivar scorer: High-level async interface for scorer management + :vartype scorer: AsyncScorerOps :ivar snapshot: High-level async interface for snapshot management :vartype snapshot: AsyncSnapshotOps :ivar storage_object: High-level async interface for storage object management @@ -504,6 +570,7 @@ class AsyncRunloopSDK: api: AsyncRunloop devbox: AsyncDevboxOps blueprint: AsyncBlueprintOps + scorer: AsyncScorerOps snapshot: AsyncSnapshotOps storage_object: AsyncStorageObjectOps @@ -547,6 +614,7 @@ def __init__( self.devbox = AsyncDevboxOps(self.api) self.blueprint = AsyncBlueprintOps(self.api) + self.scorer = AsyncScorerOps(self.api) self.snapshot = AsyncSnapshotOps(self.api) self.storage_object = AsyncStorageObjectOps(self.api) diff --git a/src/runloop_api_client/sdk/async_scorer.py b/src/runloop_api_client/sdk/async_scorer.py index a48dc81d2..848d6d0f2 100644 --- a/src/runloop_api_client/sdk/async_scorer.py +++ b/src/runloop_api_client/sdk/async_scorer.py @@ -6,8 +6,6 @@ from ._types import ( BaseRequestOptions, - SDKScorerListParams, - SDKScorerCreateParams, SDKScorerUpdateParams, SDKScorerValidateParams, ) @@ -46,44 +44,6 @@ def id(self) -> str: """ return self._id - # TODO: replace static method once we have a proper client - @staticmethod - async def create( - client: AsyncRunloop, - **params: Unpack[SDKScorerCreateParams], - ) -> "AsyncScorer": - """Create a new scenario scorer. - - :param client: Generated AsyncRunloop client - :type client: AsyncRunloop - :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters - :return: Wrapper bound to the newly created scorer - :rtype: AsyncScorer - """ - response = await client.scenarios.scorers.create( - **params, - ) - return AsyncScorer(client, response.id) - - # TODO: replace static method once we have a proper client - @staticmethod - async def list( - client: AsyncRunloop, - **params: Unpack[SDKScorerListParams], - ) -> list["AsyncScorer"]: - """List all scenario scorers. - - :param client: Generated AsyncRunloop client - :type client: AsyncRunloop - :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters - :return: List of scorer wrappers - :rtype: list[AsyncScorer] - """ - page = await client.scenarios.scorers.list( - **params, - ) - return [AsyncScorer(client, item.id) async for item in page] - async def get_info( self, **options: Unpack[BaseRequestOptions], diff --git a/src/runloop_api_client/sdk/scorer.py b/src/runloop_api_client/sdk/scorer.py index 2140bada1..6063bf0c6 100644 --- a/src/runloop_api_client/sdk/scorer.py +++ b/src/runloop_api_client/sdk/scorer.py @@ -6,8 +6,6 @@ from ._types import ( BaseRequestOptions, - SDKScorerListParams, - SDKScorerCreateParams, SDKScorerUpdateParams, SDKScorerValidateParams, ) @@ -46,44 +44,6 @@ def id(self) -> str: """ return self._id - # TODO: replace static method once we have a proper client - @staticmethod - def create( - client: Runloop, - **params: Unpack[SDKScorerCreateParams], - ) -> "Scorer": - """Create a new scenario scorer. - - :param client: Generated Runloop client - :type client: Runloop - :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters - :return: Wrapper bound to the newly created scorer - :rtype: Scorer - """ - response = client.scenarios.scorers.create( - **params, - ) - return Scorer(client, response.id) - - # TODO: replace static method once we have a proper client - @staticmethod - def list( - client: Runloop, - **params: Unpack[SDKScorerListParams], - ) -> list["Scorer"]: - """List all scenario scorers. - - :param client: Generated Runloop client - :type client: Runloop - :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters - :return: List of scorer wrappers - :rtype: list[Scorer] - """ - page = client.scenarios.scorers.list( - **params, - ) - return [Scorer(client, item.id) for item in page] - def get_info( self, **options: Unpack[BaseRequestOptions], diff --git a/src/runloop_api_client/sdk/sync.py b/src/runloop_api_client/sdk/sync.py index 94715cca4..b90549497 100644 --- a/src/runloop_api_client/sdk/sync.py +++ b/src/runloop_api_client/sdk/sync.py @@ -15,14 +15,17 @@ LongRequestOptions, SDKDevboxListParams, SDKObjectListParams, + SDKScorerListParams, SDKDevboxCreateParams, SDKObjectCreateParams, + SDKScorerCreateParams, SDKBlueprintListParams, SDKBlueprintCreateParams, SDKDiskSnapshotListParams, SDKDevboxCreateFromImageParams, ) from .devbox import Devbox +from .scorer import Scorer from .._types import Timeout, NotGiven, not_given from .._client import DEFAULT_MAX_RETRIES, Runloop from ._helpers import detect_content_type @@ -470,6 +473,67 @@ def upload_from_bytes( return obj +class ScorerOps: + """High-level manager for creating and managing scenario scorers. + + Accessed via ``runloop.scorer`` from :class:`RunloopSDK`, provides methods + to create, list, and access scorers. + + Example: + >>> runloop = RunloopSDK() + >>> scorer = runloop.scorer.create(name="my-scorer", scorer_type="llm_judge") + >>> scorers = runloop.scorer.list() + """ + + def __init__(self, client: Runloop) -> None: + """Initialize the manager. + + :param client: Generated Runloop client to wrap + :type client: Runloop + """ + self._client = client + + def create( + self, + **params: Unpack[SDKScorerCreateParams], + ) -> Scorer: + """Create a new scenario scorer. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters + :return: Wrapper bound to the newly created scorer + :rtype: Scorer + """ + response = self._client.scenarios.scorers.create( + **params, + ) + return Scorer(self._client, response.id) + + def from_id(self, scorer_id: str) -> Scorer: + """Return a scorer wrapper for the given ID. + + :param scorer_id: Scorer ID to wrap + :type scorer_id: str + :return: Wrapper for the scorer resource + :rtype: Scorer + """ + return Scorer(self._client, scorer_id) + + def list( + self, + **params: Unpack[SDKScorerListParams], + ) -> list[Scorer]: + """List all scenario scorers. + + :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters + :return: List of scorer wrappers + :rtype: list[Scorer] + """ + page = self._client.scenarios.scorers.list( + **params, + ) + return [Scorer(self._client, item.id) for item in page] + + class RunloopSDK: """High-level synchronous entry point for the Runloop SDK. @@ -483,6 +547,8 @@ class RunloopSDK: :vartype devbox: DevboxOps :ivar blueprint: High-level interface for blueprint management :vartype blueprint: BlueprintOps + :ivar scorer: High-level interface for scorer management + :vartype scorer: ScorerOps :ivar snapshot: High-level interface for snapshot management :vartype snapshot: SnapshotOps :ivar storage_object: High-level interface for storage object management @@ -499,6 +565,7 @@ class RunloopSDK: api: Runloop devbox: DevboxOps blueprint: BlueprintOps + scorer: ScorerOps snapshot: SnapshotOps storage_object: StorageObjectOps @@ -542,6 +609,7 @@ def __init__( self.devbox = DevboxOps(self.api) self.blueprint = BlueprintOps(self.api) + self.scorer = ScorerOps(self.api) self.snapshot = SnapshotOps(self.api) self.storage_object = StorageObjectOps(self.api) From d2910e7d2bfad2b361ad873760ba3ef66d81ecb0 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 14:44:13 -0500 Subject: [PATCH 03/13] fix example docstrings to use correct scorer create params --- src/runloop_api_client/sdk/async_.py | 2 +- src/runloop_api_client/sdk/sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runloop_api_client/sdk/async_.py b/src/runloop_api_client/sdk/async_.py index 9931eeb7a..1a95e296a 100644 --- a/src/runloop_api_client/sdk/async_.py +++ b/src/runloop_api_client/sdk/async_.py @@ -486,7 +486,7 @@ class AsyncScorerOps: Example: >>> runloop = AsyncRunloopSDK() - >>> scorer = await runloop.scorer.create(name="my-scorer", scorer_type="llm_judge") + >>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") >>> scorers = await runloop.scorer.list() """ diff --git a/src/runloop_api_client/sdk/sync.py b/src/runloop_api_client/sdk/sync.py index b90549497..ed1348004 100644 --- a/src/runloop_api_client/sdk/sync.py +++ b/src/runloop_api_client/sdk/sync.py @@ -481,7 +481,7 @@ class ScorerOps: Example: >>> runloop = RunloopSDK() - >>> scorer = runloop.scorer.create(name="my-scorer", scorer_type="llm_judge") + >>> scorer = runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") >>> scorers = runloop.scorer.list() """ From 4194ce1036413b9622f40a2579add4d321af4ace Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 16:02:29 -0500 Subject: [PATCH 04/13] scorer tests --- tests/sdk/conftest.py | 16 +++ tests/sdk/test_async_clients.py | 52 ++++++++- tests/sdk/test_async_scorer.py | 89 ++++++++++++++++ tests/sdk/test_clients.py | 46 +++++++- tests/sdk/test_scorer.py | 105 +++++++++++++++++++ tests/smoketests/sdk/test_async_scorer.py | 122 ++++++++++++++++++++++ tests/smoketests/sdk/test_scorer.py | 122 ++++++++++++++++++++++ 7 files changed, 550 insertions(+), 2 deletions(-) create mode 100644 tests/sdk/test_async_scorer.py create mode 100644 tests/sdk/test_scorer.py create mode 100644 tests/smoketests/sdk/test_async_scorer.py create mode 100644 tests/smoketests/sdk/test_scorer.py diff --git a/tests/sdk/conftest.py b/tests/sdk/conftest.py index 436c4de53..b61a93301 100644 --- a/tests/sdk/conftest.py +++ b/tests/sdk/conftest.py @@ -20,6 +20,7 @@ "snapshot": "snap_123", "blueprint": "bp_123", "object": "obj_123", + "scorer": "scorer_123", } # Test URL constants @@ -86,6 +87,15 @@ class MockObjectView: name: str = "test-object" +@dataclass +class MockScorerView: + """Mock ScorerView for testing.""" + + id: str = "scorer_123" + bash_script: str = "echo 'score=1.0'" + type: str = "test_scorer" + + def create_mock_httpx_client(methods: dict[str, Any] | None = None) -> AsyncMock: """ Create a mock httpx.AsyncClient with proper context manager setup. @@ -170,6 +180,12 @@ def object_view() -> MockObjectView: return MockObjectView() +@pytest.fixture +def scorer_view() -> MockScorerView: + """Create a mock ScorerView.""" + return MockScorerView() + + @pytest.fixture def mock_httpx_response() -> Mock: """Create a mock httpx.Response.""" diff --git a/tests/sdk/test_async_clients.py b/tests/sdk/test_async_clients.py index ebc8aba4a..3777c71c6 100644 --- a/tests/sdk/test_async_clients.py +++ b/tests/sdk/test_async_clients.py @@ -13,13 +13,15 @@ from tests.sdk.conftest import ( MockDevboxView, MockObjectView, + MockScorerView, MockSnapshotView, MockBlueprintView, create_mock_httpx_response, ) -from runloop_api_client.sdk import AsyncDevbox, AsyncSnapshot, AsyncBlueprint, AsyncStorageObject +from runloop_api_client.sdk import AsyncDevbox, AsyncScorer, AsyncSnapshot, AsyncBlueprint, AsyncStorageObject from runloop_api_client.sdk.async_ import ( AsyncDevboxOps, + AsyncScorerOps, AsyncRunloopSDK, AsyncSnapshotOps, AsyncBlueprintOps, @@ -515,6 +517,53 @@ async def test_upload_from_dir_with_string_path( mock_async_client.objects.complete.assert_awaited_once() +class TestAsyncScorerClient: + """Tests for AsyncScorerClient class.""" + + @pytest.mark.asyncio + async def test_create(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: + """Test create method.""" + mock_async_client.scenarios.scorers.create = AsyncMock(return_value=scorer_view) + + client = AsyncScorerOps(mock_async_client) + scorer = await client.create( + bash_script="echo 'score=1.0'", + type="test_scorer", + ) + + assert isinstance(scorer, AsyncScorer) + assert scorer.id == "scorer_123" + mock_async_client.scenarios.scorers.create.assert_called_once() + + def test_from_id(self, mock_async_client: AsyncMock) -> None: + """Test from_id method.""" + client = AsyncScorerOps(mock_async_client) + scorer = client.from_id("scorer_123") + + assert isinstance(scorer, AsyncScorer) + assert scorer.id == "scorer_123" + + @pytest.mark.asyncio + async def test_list(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: + """Test list method.""" + + async def async_iter(): + yield scorer_view + + mock_async_client.scenarios.scorers.list = AsyncMock(return_value=async_iter()) + + client = AsyncScorerOps(mock_async_client) + scorers = await client.list( + limit=10, + starting_after="scorer_000", + ) + + assert len(scorers) == 1 + assert isinstance(scorers[0], AsyncScorer) + assert scorers[0].id == "scorer_123" + mock_async_client.scenarios.scorers.list.assert_called_once() + + class TestAsyncRunloopSDK: """Tests for AsyncRunloopSDK class.""" @@ -523,6 +572,7 @@ def test_init(self) -> None: sdk = AsyncRunloopSDK(bearer_token="test-token") assert sdk.api is not None assert isinstance(sdk.devbox, AsyncDevboxOps) + assert isinstance(sdk.scorer, AsyncScorerOps) assert isinstance(sdk.snapshot, AsyncSnapshotOps) assert isinstance(sdk.blueprint, AsyncBlueprintOps) assert isinstance(sdk.storage_object, AsyncStorageObjectOps) diff --git a/tests/sdk/test_async_scorer.py b/tests/sdk/test_async_scorer.py new file mode 100644 index 000000000..622c27d1b --- /dev/null +++ b/tests/sdk/test_async_scorer.py @@ -0,0 +1,89 @@ +"""Comprehensive tests for async AsyncScorer class.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from tests.sdk.conftest import MockScorerView +from runloop_api_client.sdk import AsyncScorer + + +class TestAsyncScorer: + """Tests for AsyncScorer class.""" + + def test_init(self, mock_async_client: AsyncMock) -> None: + """Test AsyncScorer initialization.""" + scorer = AsyncScorer(mock_async_client, "scorer_123") + assert scorer.id == "scorer_123" + + def test_repr(self, mock_async_client: AsyncMock) -> None: + """Test AsyncScorer string representation.""" + scorer = AsyncScorer(mock_async_client, "scorer_123") + assert repr(scorer) == "" + + def test_id_property(self, mock_async_client: AsyncMock) -> None: + """Test id property returns the scorer ID.""" + scorer = AsyncScorer(mock_async_client, "scorer_123") + assert scorer.id == "scorer_123" + + @pytest.mark.asyncio + async def test_get_info(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: + """Test get_info method.""" + mock_async_client.scenarios.scorers.retrieve = AsyncMock(return_value=scorer_view) + + scorer = AsyncScorer(mock_async_client, "scorer_123") + result = await scorer.get_info( + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + assert result == scorer_view + mock_async_client.scenarios.scorers.retrieve.assert_called_once() + + @pytest.mark.asyncio + async def test_update(self, mock_async_client: AsyncMock) -> None: + """Test update method.""" + update_response = SimpleNamespace(id="scorer_123", name="updated-scorer") + mock_async_client.scenarios.scorers.update = AsyncMock(return_value=update_response) + + scorer = AsyncScorer(mock_async_client, "scorer_123") + result = await scorer.update( + name="updated-scorer", + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + assert result == update_response + mock_async_client.scenarios.scorers.update.assert_called_once() + + @pytest.mark.asyncio + async def test_validate(self, mock_async_client: AsyncMock) -> None: + """Test validate method.""" + validate_response = SimpleNamespace( + is_valid=True, + score=0.95, + reasoning="The output matches expected criteria.", + ) + mock_async_client.scenarios.scorers.validate = AsyncMock(return_value=validate_response) + + scorer = AsyncScorer(mock_async_client, "scorer_123") + result = await scorer.validate( + bash_command_output="test output", + expected_output="test output", + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + assert result == validate_response + assert result.is_valid is True + assert result.score == 0.95 + mock_async_client.scenarios.scorers.validate.assert_called_once() diff --git a/tests/sdk/test_clients.py b/tests/sdk/test_clients.py index 18e1342e4..ba6655eef 100644 --- a/tests/sdk/test_clients.py +++ b/tests/sdk/test_clients.py @@ -11,13 +11,15 @@ from tests.sdk.conftest import ( MockDevboxView, MockObjectView, + MockScorerView, MockSnapshotView, MockBlueprintView, create_mock_httpx_response, ) -from runloop_api_client.sdk import Devbox, Snapshot, Blueprint, StorageObject +from runloop_api_client.sdk import Devbox, Scorer, Snapshot, Blueprint, StorageObject from runloop_api_client.sdk.sync import ( DevboxOps, + ScorerOps, RunloopSDK, SnapshotOps, BlueprintOps, @@ -472,6 +474,47 @@ def test_upload_from_dir_with_string_path( mock_client.objects.complete.assert_called_once() +class TestScorerClient: + """Tests for ScorerClient class.""" + + def test_create(self, mock_client: Mock, scorer_view: MockScorerView) -> None: + """Test create method.""" + mock_client.scenarios.scorers.create.return_value = scorer_view + + client = ScorerOps(mock_client) + scorer = client.create( + bash_script="echo 'score=1.0'", + type="test_scorer", + ) + + assert isinstance(scorer, Scorer) + assert scorer.id == "scorer_123" + mock_client.scenarios.scorers.create.assert_called_once() + + def test_from_id(self, mock_client: Mock) -> None: + """Test from_id method.""" + client = ScorerOps(mock_client) + scorer = client.from_id("scorer_123") + + assert isinstance(scorer, Scorer) + assert scorer.id == "scorer_123" + + def test_list(self, mock_client: Mock, scorer_view: MockScorerView) -> None: + """Test list method.""" + mock_client.scenarios.scorers.list.return_value = [scorer_view] + + client = ScorerOps(mock_client) + scorers = client.list( + limit=10, + starting_after="scorer_000", + ) + + assert len(scorers) == 1 + assert isinstance(scorers[0], Scorer) + assert scorers[0].id == "scorer_123" + mock_client.scenarios.scorers.list.assert_called_once() + + class TestRunloopSDK: """Tests for RunloopSDK class.""" @@ -480,6 +523,7 @@ def test_init(self) -> None: sdk = RunloopSDK(bearer_token="test-token") assert sdk.api is not None assert isinstance(sdk.devbox, DevboxOps) + assert isinstance(sdk.scorer, ScorerOps) assert isinstance(sdk.snapshot, SnapshotOps) assert isinstance(sdk.blueprint, BlueprintOps) assert isinstance(sdk.storage_object, StorageObjectOps) diff --git a/tests/sdk/test_scorer.py b/tests/sdk/test_scorer.py new file mode 100644 index 000000000..731eb76a6 --- /dev/null +++ b/tests/sdk/test_scorer.py @@ -0,0 +1,105 @@ +"""Comprehensive tests for sync Scorer class.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +from tests.sdk.conftest import MockScorerView +from runloop_api_client.sdk import Scorer + + +class TestScorer: + """Tests for Scorer class.""" + + def test_init(self, mock_client: Mock) -> None: + """Test Scorer initialization.""" + scorer = Scorer(mock_client, "scorer_123") + assert scorer.id == "scorer_123" + + def test_repr(self, mock_client: Mock) -> None: + """Test Scorer string representation.""" + scorer = Scorer(mock_client, "scorer_123") + assert repr(scorer) == "" + + def test_id_property(self, mock_client: Mock) -> None: + """Test id property returns the scorer ID.""" + scorer = Scorer(mock_client, "scorer_123") + assert scorer.id == "scorer_123" + + def test_get_info(self, mock_client: Mock, scorer_view: MockScorerView) -> None: + """Test get_info method.""" + mock_client.scenarios.scorers.retrieve.return_value = scorer_view + + scorer = Scorer(mock_client, "scorer_123") + result = scorer.get_info( + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + assert result == scorer_view + mock_client.scenarios.scorers.retrieve.assert_called_once_with( + "scorer_123", + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + def test_update(self, mock_client: Mock) -> None: + """Test update method.""" + update_response = SimpleNamespace(id="scorer_123", name="updated-scorer") + mock_client.scenarios.scorers.update.return_value = update_response + + scorer = Scorer(mock_client, "scorer_123") + result = scorer.update( + name="updated-scorer", + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + assert result == update_response + mock_client.scenarios.scorers.update.assert_called_once_with( + "scorer_123", + name="updated-scorer", + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + def test_validate(self, mock_client: Mock) -> None: + """Test validate method.""" + validate_response = SimpleNamespace( + is_valid=True, + score=0.95, + reasoning="The output matches expected criteria.", + ) + mock_client.scenarios.scorers.validate.return_value = validate_response + + scorer = Scorer(mock_client, "scorer_123") + result = scorer.validate( + bash_command_output="test output", + expected_output="test output", + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) + + assert result == validate_response + assert result.is_valid is True + assert result.score == 0.95 + mock_client.scenarios.scorers.validate.assert_called_once_with( + "scorer_123", + bash_command_output="test output", + expected_output="test output", + extra_headers={"X-Custom": "value"}, + extra_query={"param": "value"}, + extra_body={"key": "value"}, + timeout=30.0, + ) diff --git a/tests/smoketests/sdk/test_async_scorer.py b/tests/smoketests/sdk/test_async_scorer.py new file mode 100644 index 000000000..ce6603d64 --- /dev/null +++ b/tests/smoketests/sdk/test_async_scorer.py @@ -0,0 +1,122 @@ +"""Asynchronous SDK smoke tests for Scorer operations.""" + +from __future__ import annotations + +import pytest + +from runloop_api_client import InternalServerError +from runloop_api_client.sdk import AsyncRunloopSDK +from tests.smoketests.utils import unique_name + +pytestmark = [pytest.mark.smoketest, pytest.mark.asyncio] + +THIRTY_SECOND_TIMEOUT = 30 +ONE_MINUTE_TIMEOUT = 60 + + +class TestAsyncScorerLifecycle: + """Test basic async scorer lifecycle operations.""" + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + async def test_scorer_create_basic(self, async_sdk_client: AsyncRunloopSDK) -> None: + """Test creating a basic scorer.""" + scorer_type = unique_name("sdk-async-scorer-basic") + scorer = await async_sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=1.0'", + ) + + assert scorer is not None + assert scorer.id is not None + assert len(scorer.id) > 0 + + # Verify it's created successfully + info = await scorer.get_info() + assert info.type == scorer_type + assert info.bash_script == "echo 'score=1.0'" + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + async def test_scorer_get_info(self, async_sdk_client: AsyncRunloopSDK) -> None: + """Test retrieving scorer information.""" + scorer_type = unique_name("sdk-async-scorer-info") + scorer = await async_sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=0.5'", + ) + + info = await scorer.get_info() + + assert info.id == scorer.id + assert info.type == scorer_type + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + async def test_scorer_update(self, async_sdk_client: AsyncRunloopSDK) -> None: + """Test updating a scorer.""" + scorer_type = unique_name("sdk-async-scorer-update") + scorer = await async_sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=0.0'", + ) + + updated_type = unique_name("sdk-async-scorer-updated") + result = await scorer.update( + type=updated_type, + bash_script="echo 'score=1.0'", + ) + + assert result is not None + + # Verify the update + info = await scorer.get_info() + assert info.type == updated_type + assert info.bash_script == "echo 'score=1.0'" + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + async def test_scorer_validate(self, async_sdk_client: AsyncRunloopSDK) -> None: + """Test validating a scorer.""" + scorer_type = unique_name("sdk-async-scorer-validate") + scorer = await async_sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=1.0'", + ) + + try: + result = await scorer.validate( + scoring_context={}, + ) + assert result is not None + except InternalServerError: + # Backend may return 500 for validate endpoint - skip if this happens + pytest.skip("Backend returned 500 for scorer validate endpoint") + + +class TestAsyncScorerListing: + """Test async scorer listing and retrieval operations.""" + + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) + async def test_list_scorers(self, async_sdk_client: AsyncRunloopSDK) -> None: + """Test listing scorers.""" + scorers = await async_sdk_client.scorer.list(limit=10) + + assert isinstance(scorers, list) + # List might be empty, that's okay + assert len(scorers) >= 0 + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + async def test_get_scorer_by_id(self, async_sdk_client: AsyncRunloopSDK) -> None: + """Test retrieving scorer by ID.""" + # Create a scorer + scorer_type = unique_name("sdk-async-scorer-retrieve") + created = await async_sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=1.0'", + ) + + # Retrieve it by ID + retrieved = async_sdk_client.scorer.from_id(created.id) + assert retrieved.id == created.id + + # Verify it's the same scorer + info = await retrieved.get_info() + assert info.id == created.id + assert info.type == scorer_type diff --git a/tests/smoketests/sdk/test_scorer.py b/tests/smoketests/sdk/test_scorer.py new file mode 100644 index 000000000..01df84df9 --- /dev/null +++ b/tests/smoketests/sdk/test_scorer.py @@ -0,0 +1,122 @@ +"""Synchronous SDK smoke tests for Scorer operations.""" + +from __future__ import annotations + +import pytest + +from runloop_api_client import InternalServerError +from runloop_api_client.sdk import RunloopSDK +from tests.smoketests.utils import unique_name + +pytestmark = [pytest.mark.smoketest] + +THIRTY_SECOND_TIMEOUT = 30 +ONE_MINUTE_TIMEOUT = 60 + + +class TestScorerLifecycle: + """Test basic scorer lifecycle operations.""" + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + def test_scorer_create_basic(self, sdk_client: RunloopSDK) -> None: + """Test creating a basic scorer.""" + scorer_type = unique_name("sdk-scorer-basic") + scorer = sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=1.0'", + ) + + assert scorer is not None + assert scorer.id is not None + assert len(scorer.id) > 0 + + # Verify it's created successfully + info = scorer.get_info() + assert info.type == scorer_type + assert info.bash_script == "echo 'score=1.0'" + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + def test_scorer_get_info(self, sdk_client: RunloopSDK) -> None: + """Test retrieving scorer information.""" + scorer_type = unique_name("sdk-scorer-info") + scorer = sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=0.5'", + ) + + info = scorer.get_info() + + assert info.id == scorer.id + assert info.type == scorer_type + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + def test_scorer_update(self, sdk_client: RunloopSDK) -> None: + """Test updating a scorer.""" + scorer_type = unique_name("sdk-scorer-update") + scorer = sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=0.0'", + ) + + updated_type = unique_name("sdk-scorer-updated") + result = scorer.update( + type=updated_type, + bash_script="echo 'score=1.0'", + ) + + assert result is not None + + # Verify the update + info = scorer.get_info() + assert info.type == updated_type + assert info.bash_script == "echo 'score=1.0'" + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + def test_scorer_validate(self, sdk_client: RunloopSDK) -> None: + """Test validating a scorer.""" + scorer_type = unique_name("sdk-scorer-validate") + scorer = sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=1.0'", + ) + + try: + result = scorer.validate( + scoring_context={}, + ) + assert result is not None + except InternalServerError: + # Backend may return 500 for validate endpoint - skip if this happens + pytest.skip("Backend returned 500 for scorer validate endpoint") + + +class TestScorerListing: + """Test scorer listing and retrieval operations.""" + + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) + def test_list_scorers(self, sdk_client: RunloopSDK) -> None: + """Test listing scorers.""" + scorers = sdk_client.scorer.list(limit=10) + + assert isinstance(scorers, list) + # List might be empty, that's okay + assert len(scorers) >= 0 + + @pytest.mark.timeout(ONE_MINUTE_TIMEOUT) + def test_get_scorer_by_id(self, sdk_client: RunloopSDK) -> None: + """Test retrieving scorer by ID.""" + # Create a scorer + scorer_type = unique_name("sdk-scorer-retrieve") + created = sdk_client.scorer.create( + type=scorer_type, + bash_script="echo 'score=1.0'", + ) + + # Retrieve it by ID + retrieved = sdk_client.scorer.from_id(created.id) + assert retrieved.id == created.id + + # Verify it's the same scorer + info = retrieved.get_info() + assert info.id == created.id + assert info.type == scorer_type From e8fdc37f78f806ab20811354df59e46066177ffc Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 17:02:39 -0500 Subject: [PATCH 05/13] fixed scorer unit test parameters for update and validate --- tests/sdk/test_async_scorer.py | 16 +++++++--------- tests/sdk/test_scorer.py | 22 ++++++++++------------ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tests/sdk/test_async_scorer.py b/tests/sdk/test_async_scorer.py index 622c27d1b..5e316defd 100644 --- a/tests/sdk/test_async_scorer.py +++ b/tests/sdk/test_async_scorer.py @@ -48,12 +48,13 @@ async def test_get_info(self, mock_async_client: AsyncMock, scorer_view: MockSco @pytest.mark.asyncio async def test_update(self, mock_async_client: AsyncMock) -> None: """Test update method.""" - update_response = SimpleNamespace(id="scorer_123", name="updated-scorer") + update_response = SimpleNamespace(id="scorer_123", type="updated_scorer", bash_script="echo 'score=1.0'") mock_async_client.scenarios.scorers.update = AsyncMock(return_value=update_response) scorer = AsyncScorer(mock_async_client, "scorer_123") result = await scorer.update( - name="updated-scorer", + type="updated_scorer", + bash_script="echo 'score=1.0'", extra_headers={"X-Custom": "value"}, extra_query={"param": "value"}, extra_body={"key": "value"}, @@ -67,16 +68,15 @@ async def test_update(self, mock_async_client: AsyncMock) -> None: async def test_validate(self, mock_async_client: AsyncMock) -> None: """Test validate method.""" validate_response = SimpleNamespace( - is_valid=True, - score=0.95, - reasoning="The output matches expected criteria.", + name="test_scorer", + scoring_context={}, + scoring_result=SimpleNamespace(score=0.95), ) mock_async_client.scenarios.scorers.validate = AsyncMock(return_value=validate_response) scorer = AsyncScorer(mock_async_client, "scorer_123") result = await scorer.validate( - bash_command_output="test output", - expected_output="test output", + scoring_context={"test": "context"}, extra_headers={"X-Custom": "value"}, extra_query={"param": "value"}, extra_body={"key": "value"}, @@ -84,6 +84,4 @@ async def test_validate(self, mock_async_client: AsyncMock) -> None: ) assert result == validate_response - assert result.is_valid is True - assert result.score == 0.95 mock_async_client.scenarios.scorers.validate.assert_called_once() diff --git a/tests/sdk/test_scorer.py b/tests/sdk/test_scorer.py index 731eb76a6..81ebdd9f3 100644 --- a/tests/sdk/test_scorer.py +++ b/tests/sdk/test_scorer.py @@ -50,12 +50,13 @@ def test_get_info(self, mock_client: Mock, scorer_view: MockScorerView) -> None: def test_update(self, mock_client: Mock) -> None: """Test update method.""" - update_response = SimpleNamespace(id="scorer_123", name="updated-scorer") + update_response = SimpleNamespace(id="scorer_123", type="updated_scorer", bash_script="echo 'score=1.0'") mock_client.scenarios.scorers.update.return_value = update_response scorer = Scorer(mock_client, "scorer_123") result = scorer.update( - name="updated-scorer", + type="updated_scorer", + bash_script="echo 'score=1.0'", extra_headers={"X-Custom": "value"}, extra_query={"param": "value"}, extra_body={"key": "value"}, @@ -65,7 +66,8 @@ def test_update(self, mock_client: Mock) -> None: assert result == update_response mock_client.scenarios.scorers.update.assert_called_once_with( "scorer_123", - name="updated-scorer", + type="updated_scorer", + bash_script="echo 'score=1.0'", extra_headers={"X-Custom": "value"}, extra_query={"param": "value"}, extra_body={"key": "value"}, @@ -75,16 +77,15 @@ def test_update(self, mock_client: Mock) -> None: def test_validate(self, mock_client: Mock) -> None: """Test validate method.""" validate_response = SimpleNamespace( - is_valid=True, - score=0.95, - reasoning="The output matches expected criteria.", + name="test_scorer", + scoring_context={}, + scoring_result=SimpleNamespace(score=0.95), ) mock_client.scenarios.scorers.validate.return_value = validate_response scorer = Scorer(mock_client, "scorer_123") result = scorer.validate( - bash_command_output="test output", - expected_output="test output", + scoring_context={"test": "context"}, extra_headers={"X-Custom": "value"}, extra_query={"param": "value"}, extra_body={"key": "value"}, @@ -92,12 +93,9 @@ def test_validate(self, mock_client: Mock) -> None: ) assert result == validate_response - assert result.is_valid is True - assert result.score == 0.95 mock_client.scenarios.scorers.validate.assert_called_once_with( "scorer_123", - bash_command_output="test output", - expected_output="test output", + scoring_context={"test": "context"}, extra_headers={"X-Custom": "value"}, extra_query={"param": "value"}, extra_body={"key": "value"}, From c1f617e66848248d734562fc9b90076283afa86a Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 17:42:22 -0500 Subject: [PATCH 06/13] update scorer and scorer ops docstrings to be more helpful while not exposing system internals --- src/runloop_api_client/sdk/async_.py | 43 +++++-------- src/runloop_api_client/sdk/async_scorer.py | 71 +++++++++------------- src/runloop_api_client/sdk/scorer.py | 71 +++++++++------------- src/runloop_api_client/sdk/sync.py | 43 +++++-------- 4 files changed, 88 insertions(+), 140 deletions(-) diff --git a/src/runloop_api_client/sdk/async_.py b/src/runloop_api_client/sdk/async_.py index 1a95e296a..86338f0c1 100644 --- a/src/runloop_api_client/sdk/async_.py +++ b/src/runloop_api_client/sdk/async_.py @@ -479,63 +479,50 @@ async def upload_from_bytes( class AsyncScorerOps: - """High-level async manager for creating and managing scenario scorers. - - Accessed via ``runloop.scorer`` from :class:`AsyncRunloopSDK`, provides - coroutines to create, list, and access scorers. + """Create and manage custom scorers (async). Access via ``runloop.scorer``. Example: >>> runloop = AsyncRunloopSDK() >>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") - >>> scorers = await runloop.scorer.list() + >>> all_scorers = await runloop.scorer.list() """ def __init__(self, client: AsyncRunloop) -> None: - """Initialize the manager. + """Initialize AsyncScorerOps. - :param client: Generated AsyncRunloop client to wrap + :param client: AsyncRunloop client instance :type client: AsyncRunloop """ self._client = client - async def create( - self, - **params: Unpack[SDKScorerCreateParams], - ) -> AsyncScorer: - """Create a new scenario scorer. + async def create(self, **params: Unpack[SDKScorerCreateParams]) -> AsyncScorer: + """Create a new scorer with the given type and bash script. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters - :return: Wrapper bound to the newly created scorer + :return: The newly created scorer :rtype: AsyncScorer """ - response = await self._client.scenarios.scorers.create( - **params, - ) + response = await self._client.scenarios.scorers.create(**params) return AsyncScorer(self._client, response.id) def from_id(self, scorer_id: str) -> AsyncScorer: - """Return a scorer wrapper for the given ID. + """Get an AsyncScorer instance for an existing scorer ID. - :param scorer_id: Scorer ID to wrap + :param scorer_id: ID of the scorer :type scorer_id: str - :return: Wrapper for the scorer resource + :return: AsyncScorer instance for the given ID :rtype: AsyncScorer """ return AsyncScorer(self._client, scorer_id) - async def list( - self, - **params: Unpack[SDKScorerListParams], - ) -> list[AsyncScorer]: - """List all scenario scorers. + async def list(self, **params: Unpack[SDKScorerListParams]) -> list[AsyncScorer]: + """List all scorers, optionally filtered by parameters. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters - :return: List of scorer wrappers + :return: List of scorers :rtype: list[AsyncScorer] """ - page = await self._client.scenarios.scorers.list( - **params, - ) + page = await self._client.scenarios.scorers.list(**params) return [AsyncScorer(self._client, item.id) async for item in page] diff --git a/src/runloop_api_client/sdk/async_scorer.py b/src/runloop_api_client/sdk/async_scorer.py index 848d6d0f2..3df4fb4e0 100644 --- a/src/runloop_api_client/sdk/async_scorer.py +++ b/src/runloop_api_client/sdk/async_scorer.py @@ -14,18 +14,23 @@ class AsyncScorer: - """Asynchronous wrapper around a scenario scorer resource.""" + """A custom scorer for evaluating scenario outputs (async). - def __init__( - self, - client: AsyncRunloop, - scorer_id: str, - ) -> None: - """Initialize the wrapper. + Scorers define bash scripts that produce a score (0.0-1.0) for scenario runs. + Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``. - :param client: Generated AsyncRunloop client + Example: + >>> runloop = AsyncRunloopSDK() + >>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") + >>> await scorer.validate(scoring_context={"output": "test"}) + """ + + def __init__(self, client: AsyncRunloop, scorer_id: str) -> None: + """Create an AsyncScorer instance. + + :param client: AsyncRunloop client instance :type client: AsyncRunloop - :param scorer_id: Scorer ID returned by the API + :param scorer_id: ID of the scorer :type scorer_id: str """ self._client = client @@ -37,54 +42,36 @@ def __repr__(self) -> str: @property def id(self) -> str: - """Return the scorer ID. + """The scorer's unique identifier. - :return: Unique scorer ID + :return: Scorer ID :rtype: str """ return self._id - async def get_info( - self, - **options: Unpack[BaseRequestOptions], - ) -> ScorerRetrieveResponse: - """Retrieve the latest scorer details. + async def get_info(self, **options: Unpack[BaseRequestOptions]) -> ScorerRetrieveResponse: + """Fetch current scorer details from the API. - :param options: Optional request configuration - :return: API response describing the scorer + :param options: See :typeddict:`~runloop_api_client.sdk._types.BaseRequestOptions` for available options + :return: Current scorer details :rtype: ScorerRetrieveResponse """ - return await self._client.scenarios.scorers.retrieve( - self._id, - **options, - ) + return await self._client.scenarios.scorers.retrieve(self._id, **options) - async def update( - self, - **params: Unpack[SDKScorerUpdateParams], - ) -> ScorerUpdateResponse: - """Update the scorer. + async def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateResponse: + """Update the scorer's type or bash script. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerUpdateParams` for available parameters - :return: API response with updated scorer details + :return: Updated scorer details :rtype: ScorerUpdateResponse """ - return await self._client.scenarios.scorers.update( - self._id, - **params, - ) + return await self._client.scenarios.scorers.update(self._id, **params) - async def validate( - self, - **params: Unpack[SDKScorerValidateParams], - ) -> ScorerValidateResponse: - """Validate the scorer with a given context. + async def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse: + """Run the scorer against the provided context and return the result. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters - :return: API response with validation results + :return: Validation result with score :rtype: ScorerValidateResponse """ - return await self._client.scenarios.scorers.validate( - self._id, - **params, - ) + return await self._client.scenarios.scorers.validate(self._id, **params) diff --git a/src/runloop_api_client/sdk/scorer.py b/src/runloop_api_client/sdk/scorer.py index 6063bf0c6..a25bb44a8 100644 --- a/src/runloop_api_client/sdk/scorer.py +++ b/src/runloop_api_client/sdk/scorer.py @@ -14,18 +14,23 @@ class Scorer: - """Synchronous wrapper around a scenario scorer resource.""" + """A custom scorer for evaluating scenario outputs. - def __init__( - self, - client: Runloop, - scorer_id: str, - ) -> None: - """Initialize the wrapper. + Scorers define bash scripts that produce a score (0.0-1.0) for scenario runs. + Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``. - :param client: Generated Runloop client + Example: + >>> runloop = RunloopSDK() + >>> scorer = runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") + >>> scorer.validate(scoring_context={"output": "test"}) + """ + + def __init__(self, client: Runloop, scorer_id: str) -> None: + """Create a Scorer instance. + + :param client: Runloop client instance :type client: Runloop - :param scorer_id: Scorer ID returned by the API + :param scorer_id: ID of the scorer :type scorer_id: str """ self._client = client @@ -37,54 +42,36 @@ def __repr__(self) -> str: @property def id(self) -> str: - """Return the scorer ID. + """The scorer's unique identifier. - :return: Unique scorer ID + :return: Scorer ID :rtype: str """ return self._id - def get_info( - self, - **options: Unpack[BaseRequestOptions], - ) -> ScorerRetrieveResponse: - """Retrieve the latest scorer details. + def get_info(self, **options: Unpack[BaseRequestOptions]) -> ScorerRetrieveResponse: + """Fetch current scorer details from the API. - :param options: Optional request configuration - :return: API response describing the scorer + :param options: See :typeddict:`~runloop_api_client.sdk._types.BaseRequestOptions` for available options + :return: Current scorer details :rtype: ScorerRetrieveResponse """ - return self._client.scenarios.scorers.retrieve( - self._id, - **options, - ) + return self._client.scenarios.scorers.retrieve(self._id, **options) - def update( - self, - **params: Unpack[SDKScorerUpdateParams], - ) -> ScorerUpdateResponse: - """Update the scorer. + def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateResponse: + """Update the scorer's type or bash script. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerUpdateParams` for available parameters - :return: API response with updated scorer details + :return: Updated scorer details :rtype: ScorerUpdateResponse """ - return self._client.scenarios.scorers.update( - self._id, - **params, - ) + return self._client.scenarios.scorers.update(self._id, **params) - def validate( - self, - **params: Unpack[SDKScorerValidateParams], - ) -> ScorerValidateResponse: - """Validate the scorer with a given context. + def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse: + """Run the scorer against the provided context and return the result. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters - :return: API response with validation results + :return: Validation result with score :rtype: ScorerValidateResponse """ - return self._client.scenarios.scorers.validate( - self._id, - **params, - ) + return self._client.scenarios.scorers.validate(self._id, **params) diff --git a/src/runloop_api_client/sdk/sync.py b/src/runloop_api_client/sdk/sync.py index ed1348004..703942804 100644 --- a/src/runloop_api_client/sdk/sync.py +++ b/src/runloop_api_client/sdk/sync.py @@ -474,63 +474,50 @@ def upload_from_bytes( class ScorerOps: - """High-level manager for creating and managing scenario scorers. - - Accessed via ``runloop.scorer`` from :class:`RunloopSDK`, provides methods - to create, list, and access scorers. + """Create and manage custom scorers. Access via ``runloop.scorer``. Example: >>> runloop = RunloopSDK() >>> scorer = runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") - >>> scorers = runloop.scorer.list() + >>> all_scorers = runloop.scorer.list() """ def __init__(self, client: Runloop) -> None: - """Initialize the manager. + """Initialize ScorerOps. - :param client: Generated Runloop client to wrap + :param client: Runloop client instance :type client: Runloop """ self._client = client - def create( - self, - **params: Unpack[SDKScorerCreateParams], - ) -> Scorer: - """Create a new scenario scorer. + def create(self, **params: Unpack[SDKScorerCreateParams]) -> Scorer: + """Create a new scorer with the given type and bash script. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters - :return: Wrapper bound to the newly created scorer + :return: The newly created scorer :rtype: Scorer """ - response = self._client.scenarios.scorers.create( - **params, - ) + response = self._client.scenarios.scorers.create(**params) return Scorer(self._client, response.id) def from_id(self, scorer_id: str) -> Scorer: - """Return a scorer wrapper for the given ID. + """Get a Scorer instance for an existing scorer ID. - :param scorer_id: Scorer ID to wrap + :param scorer_id: ID of the scorer :type scorer_id: str - :return: Wrapper for the scorer resource + :return: Scorer instance for the given ID :rtype: Scorer """ return Scorer(self._client, scorer_id) - def list( - self, - **params: Unpack[SDKScorerListParams], - ) -> list[Scorer]: - """List all scenario scorers. + def list(self, **params: Unpack[SDKScorerListParams]) -> list[Scorer]: + """List all scorers, optionally filtered by parameters. :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters - :return: List of scorer wrappers + :return: List of scorers :rtype: list[Scorer] """ - page = self._client.scenarios.scorers.list( - **params, - ) + page = self._client.scenarios.scorers.list(**params) return [Scorer(self._client, item.id) for item in page] From 74ca76ca2b148804f85a4f79b4b4f8d59bc3feb0 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 18:02:52 -0500 Subject: [PATCH 07/13] update docs with scorer classes, methods and types --- docs/sdk/async/index.rst | 1 + docs/sdk/async/scorer.rst | 9 +++++++++ docs/sdk/sync/index.rst | 1 + docs/sdk/sync/scorer.rst | 9 +++++++++ docs/sdk/types.rst | 13 +++++++++++++ 5 files changed, 33 insertions(+) create mode 100644 docs/sdk/async/scorer.rst create mode 100644 docs/sdk/sync/scorer.rst diff --git a/docs/sdk/async/index.rst b/docs/sdk/async/index.rst index 1d92ea76f..0ea16d5e4 100644 --- a/docs/sdk/async/index.rst +++ b/docs/sdk/async/index.rst @@ -26,4 +26,5 @@ Asynchronous resource classes for working with devboxes, blueprints, snapshots, blueprint snapshot storage_object + scorer diff --git a/docs/sdk/async/scorer.rst b/docs/sdk/async/scorer.rst new file mode 100644 index 000000000..7564092d8 --- /dev/null +++ b/docs/sdk/async/scorer.rst @@ -0,0 +1,9 @@ +Scorer +====== + +The ``AsyncScorer`` class provides asynchronous methods for managing custom scenario scorers. + +.. automodule:: runloop_api_client.sdk.async_scorer + :members: + + diff --git a/docs/sdk/sync/index.rst b/docs/sdk/sync/index.rst index c77646f2a..e7d1ca616 100644 --- a/docs/sdk/sync/index.rst +++ b/docs/sdk/sync/index.rst @@ -26,4 +26,5 @@ Synchronous resource classes for working with devboxes, blueprints, snapshots, a blueprint snapshot storage_object + scorer diff --git a/docs/sdk/sync/scorer.rst b/docs/sdk/sync/scorer.rst new file mode 100644 index 000000000..09b98dfb2 --- /dev/null +++ b/docs/sdk/sync/scorer.rst @@ -0,0 +1,9 @@ +Scorer +====== + +The ``Scorer`` class provides synchronous methods for managing custom scenario scorers. + +.. automodule:: runloop_api_client.sdk.scorer + :members: + + diff --git a/docs/sdk/types.rst b/docs/sdk/types.rst index 4de60fb5e..9d2983cd0 100644 --- a/docs/sdk/types.rst +++ b/docs/sdk/types.rst @@ -78,6 +78,19 @@ These TypeDicts define parameters for storage object creation, listing, and down .. autotypeddict:: runloop_api_client.sdk._types.SDKObjectDownloadParams +Scorer Parameters +----------------- + +These TypeDicts define parameters for scorer creation, listing, updating, and validation. + +.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerCreateParams + +.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerListParams + +.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerUpdateParams + +.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerValidateParams + Core Request Options -------------------- From c5bc7d39b98595a24abbf8b1ac75915d9e5f47c4 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 18:50:49 -0500 Subject: [PATCH 08/13] remove verbose request options in unit test parameters --- tests/sdk/test_async_scorer.py | 15 +-------------- tests/sdk/test_scorer.py | 31 ++----------------------------- 2 files changed, 3 insertions(+), 43 deletions(-) diff --git a/tests/sdk/test_async_scorer.py b/tests/sdk/test_async_scorer.py index 5e316defd..4cee049a3 100644 --- a/tests/sdk/test_async_scorer.py +++ b/tests/sdk/test_async_scorer.py @@ -35,12 +35,7 @@ async def test_get_info(self, mock_async_client: AsyncMock, scorer_view: MockSco mock_async_client.scenarios.scorers.retrieve = AsyncMock(return_value=scorer_view) scorer = AsyncScorer(mock_async_client, "scorer_123") - result = await scorer.get_info( - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, - ) + result = await scorer.get_info() assert result == scorer_view mock_async_client.scenarios.scorers.retrieve.assert_called_once() @@ -55,10 +50,6 @@ async def test_update(self, mock_async_client: AsyncMock) -> None: result = await scorer.update( type="updated_scorer", bash_script="echo 'score=1.0'", - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, ) assert result == update_response @@ -77,10 +68,6 @@ async def test_validate(self, mock_async_client: AsyncMock) -> None: scorer = AsyncScorer(mock_async_client, "scorer_123") result = await scorer.validate( scoring_context={"test": "context"}, - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, ) assert result == validate_response diff --git a/tests/sdk/test_scorer.py b/tests/sdk/test_scorer.py index 81ebdd9f3..6947e0f08 100644 --- a/tests/sdk/test_scorer.py +++ b/tests/sdk/test_scorer.py @@ -32,21 +32,10 @@ def test_get_info(self, mock_client: Mock, scorer_view: MockScorerView) -> None: mock_client.scenarios.scorers.retrieve.return_value = scorer_view scorer = Scorer(mock_client, "scorer_123") - result = scorer.get_info( - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, - ) + result = scorer.get_info() assert result == scorer_view - mock_client.scenarios.scorers.retrieve.assert_called_once_with( - "scorer_123", - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, - ) + mock_client.scenarios.scorers.retrieve.assert_called_once_with("scorer_123") def test_update(self, mock_client: Mock) -> None: """Test update method.""" @@ -57,10 +46,6 @@ def test_update(self, mock_client: Mock) -> None: result = scorer.update( type="updated_scorer", bash_script="echo 'score=1.0'", - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, ) assert result == update_response @@ -68,10 +53,6 @@ def test_update(self, mock_client: Mock) -> None: "scorer_123", type="updated_scorer", bash_script="echo 'score=1.0'", - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, ) def test_validate(self, mock_client: Mock) -> None: @@ -86,18 +67,10 @@ def test_validate(self, mock_client: Mock) -> None: scorer = Scorer(mock_client, "scorer_123") result = scorer.validate( scoring_context={"test": "context"}, - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, ) assert result == validate_response mock_client.scenarios.scorers.validate.assert_called_once_with( "scorer_123", scoring_context={"test": "context"}, - extra_headers={"X-Custom": "value"}, - extra_query={"param": "value"}, - extra_body={"key": "value"}, - timeout=30.0, ) From e3e2c14ee03c73017f7009441e260320f39fc0d0 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 18:51:19 -0500 Subject: [PATCH 09/13] rename client to ops in client test --- tests/sdk/test_async_clients.py | 104 ++++++++++++++++---------------- tests/sdk/test_clients.py | 104 ++++++++++++++++---------------- 2 files changed, 104 insertions(+), 104 deletions(-) diff --git a/tests/sdk/test_async_clients.py b/tests/sdk/test_async_clients.py index 3777c71c6..fd57a253e 100644 --- a/tests/sdk/test_async_clients.py +++ b/tests/sdk/test_async_clients.py @@ -38,8 +38,8 @@ async def test_create(self, mock_async_client: AsyncMock, devbox_view: MockDevbo """Test create method.""" mock_async_client.devboxes.create_and_await_running = AsyncMock(return_value=devbox_view) - client = AsyncDevboxOps(mock_async_client) - devbox = await client.create( + ops = AsyncDevboxOps(mock_async_client) + devbox = await ops.create( name="test-devbox", metadata={"key": "value"}, polling_config=PollingConfig(timeout_seconds=60.0), @@ -54,8 +54,8 @@ async def test_create_from_blueprint_id(self, mock_async_client: AsyncMock, devb """Test create_from_blueprint_id method.""" mock_async_client.devboxes.create_and_await_running = AsyncMock(return_value=devbox_view) - client = AsyncDevboxOps(mock_async_client) - devbox = await client.create_from_blueprint_id( + ops = AsyncDevboxOps(mock_async_client) + devbox = await ops.create_from_blueprint_id( "bp_123", name="test-devbox", ) @@ -69,8 +69,8 @@ async def test_create_from_blueprint_name(self, mock_async_client: AsyncMock, de """Test create_from_blueprint_name method.""" mock_async_client.devboxes.create_and_await_running = AsyncMock(return_value=devbox_view) - client = AsyncDevboxOps(mock_async_client) - devbox = await client.create_from_blueprint_name( + ops = AsyncDevboxOps(mock_async_client) + devbox = await ops.create_from_blueprint_name( "my-blueprint", name="test-devbox", ) @@ -84,8 +84,8 @@ async def test_create_from_snapshot(self, mock_async_client: AsyncMock, devbox_v """Test create_from_snapshot method.""" mock_async_client.devboxes.create_and_await_running = AsyncMock(return_value=devbox_view) - client = AsyncDevboxOps(mock_async_client) - devbox = await client.create_from_snapshot( + ops = AsyncDevboxOps(mock_async_client) + devbox = await ops.create_from_snapshot( "snap_123", name="test-devbox", ) @@ -96,8 +96,8 @@ async def test_create_from_snapshot(self, mock_async_client: AsyncMock, devbox_v def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" - client = AsyncDevboxOps(mock_async_client) - devbox = client.from_id("dev_123") + ops = AsyncDevboxOps(mock_async_client) + devbox = ops.from_id("dev_123") assert isinstance(devbox, AsyncDevbox) assert devbox.id == "dev_123" @@ -111,8 +111,8 @@ async def test_list(self, mock_async_client: AsyncMock, devbox_view: MockDevboxV page = SimpleNamespace(devboxes=[devbox_view]) mock_async_client.devboxes.list = AsyncMock(return_value=page) - client = AsyncDevboxOps(mock_async_client) - devboxes = await client.list( + ops = AsyncDevboxOps(mock_async_client) + devboxes = await ops.list( limit=10, status="running", starting_after="dev_000", @@ -133,8 +133,8 @@ async def test_list(self, mock_async_client: AsyncMock, snapshot_view: MockSnaps page = SimpleNamespace(snapshots=[snapshot_view]) mock_async_client.devboxes.disk_snapshots.list = AsyncMock(return_value=page) - client = AsyncSnapshotOps(mock_async_client) - snapshots = await client.list( + ops = AsyncSnapshotOps(mock_async_client) + snapshots = await ops.list( devbox_id="dev_123", limit=10, starting_after="snap_000", @@ -147,8 +147,8 @@ async def test_list(self, mock_async_client: AsyncMock, snapshot_view: MockSnaps def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" - client = AsyncSnapshotOps(mock_async_client) - snapshot = client.from_id("snap_123") + ops = AsyncSnapshotOps(mock_async_client) + snapshot = ops.from_id("snap_123") assert isinstance(snapshot, AsyncSnapshot) assert snapshot.id == "snap_123" @@ -162,8 +162,8 @@ async def test_create(self, mock_async_client: AsyncMock, blueprint_view: MockBl """Test create method.""" mock_async_client.blueprints.create_and_await_build_complete = AsyncMock(return_value=blueprint_view) - client = AsyncBlueprintOps(mock_async_client) - blueprint = await client.create( + ops = AsyncBlueprintOps(mock_async_client) + blueprint = await ops.create( name="test-blueprint", polling_config=PollingConfig(timeout_seconds=60.0), ) @@ -174,8 +174,8 @@ async def test_create(self, mock_async_client: AsyncMock, blueprint_view: MockBl def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" - client = AsyncBlueprintOps(mock_async_client) - blueprint = client.from_id("bp_123") + ops = AsyncBlueprintOps(mock_async_client) + blueprint = ops.from_id("bp_123") assert isinstance(blueprint, AsyncBlueprint) assert blueprint.id == "bp_123" @@ -186,8 +186,8 @@ async def test_list(self, mock_async_client: AsyncMock, blueprint_view: MockBlue page = SimpleNamespace(blueprints=[blueprint_view]) mock_async_client.blueprints.list = AsyncMock(return_value=page) - client = AsyncBlueprintOps(mock_async_client) - blueprints = await client.list( + ops = AsyncBlueprintOps(mock_async_client) + blueprints = await ops.list( limit=10, name="test", starting_after="bp_000", @@ -207,8 +207,8 @@ async def test_create(self, mock_async_client: AsyncMock, object_view: MockObjec """Test create method.""" mock_async_client.objects.create = AsyncMock(return_value=object_view) - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.create(name="test.txt", content_type="text", metadata={"key": "value"}) + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.create(name="test.txt", content_type="text", metadata={"key": "value"}) assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -221,8 +221,8 @@ async def test_create(self, mock_async_client: AsyncMock, object_view: MockObjec def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" - client = AsyncStorageObjectOps(mock_async_client) - obj = client.from_id("obj_123") + ops = AsyncStorageObjectOps(mock_async_client) + obj = ops.from_id("obj_123") assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -234,8 +234,8 @@ async def test_list(self, mock_async_client: AsyncMock, object_view: MockObjectV page = SimpleNamespace(objects=[object_view]) mock_async_client.objects.list = AsyncMock(return_value=page) - client = AsyncStorageObjectOps(mock_async_client) - objects = await client.list( + ops = AsyncStorageObjectOps(mock_async_client) + objects = await ops.list( content_type="text", limit=10, name="test", @@ -272,8 +272,8 @@ async def test_upload_from_file( http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.upload_from_file(temp_file, name="test.txt") + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.upload_from_file(temp_file, name="test.txt") assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -297,8 +297,8 @@ async def test_upload_from_text(self, mock_async_client: AsyncMock, object_view: http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.upload_from_text("test content", name="test.txt", metadata={"key": "value"}) + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.upload_from_text("test content", name="test.txt", metadata={"key": "value"}) assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -322,8 +322,8 @@ async def test_upload_from_bytes(self, mock_async_client: AsyncMock, object_view http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.upload_from_bytes(b"test content", name="test.bin", content_type="binary") + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.upload_from_bytes(b"test content", name="test.bin", content_type="binary") assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -339,11 +339,11 @@ async def test_upload_from_bytes(self, mock_async_client: AsyncMock, object_view @pytest.mark.asyncio async def test_upload_from_file_missing_path(self, mock_async_client: AsyncMock, tmp_path: Path) -> None: """upload_from_file should raise when file cannot be read.""" - client = AsyncStorageObjectOps(mock_async_client) + ops = AsyncStorageObjectOps(mock_async_client) missing_file = tmp_path / "missing.txt" with pytest.raises(OSError, match="Failed to read file"): - await client.upload_from_file(missing_file) + await ops.upload_from_file(missing_file) @pytest.mark.asyncio async def test_upload_from_dir( @@ -367,8 +367,8 @@ async def test_upload_from_dir( http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.upload_from_dir(test_dir, name="archive.tar.gz", metadata={"key": "value"}) + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.upload_from_dir(test_dir, name="archive.tar.gz", metadata={"key": "value"}) assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -412,8 +412,8 @@ async def test_upload_from_dir_default_name( http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.upload_from_dir(test_dir) + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.upload_from_dir(test_dir) assert isinstance(obj, AsyncStorageObject) # Name should be directory name + .tar.gz @@ -443,8 +443,8 @@ async def test_upload_from_dir_with_ttl( http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.upload_from_dir(test_dir, ttl=timedelta(hours=2)) + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.upload_from_dir(test_dir, ttl=timedelta(hours=2)) assert isinstance(obj, AsyncStorageObject) mock_async_client.objects.create.assert_awaited_once_with( @@ -470,8 +470,8 @@ async def test_upload_from_dir_empty_directory( http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) - obj = await client.upload_from_dir(test_dir) + ops = AsyncStorageObjectOps(mock_async_client) + obj = await ops.upload_from_dir(test_dir) assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -501,9 +501,9 @@ async def test_upload_from_dir_with_string_path( http_client.put = AsyncMock(return_value=mock_response) mock_async_client._client = http_client - client = AsyncStorageObjectOps(mock_async_client) + ops = AsyncStorageObjectOps(mock_async_client) # Pass string path instead of Path object - obj = await client.upload_from_dir(str(test_dir)) + obj = await ops.upload_from_dir(str(test_dir)) assert isinstance(obj, AsyncStorageObject) assert obj.id == "obj_123" @@ -525,8 +525,8 @@ async def test_create(self, mock_async_client: AsyncMock, scorer_view: MockScore """Test create method.""" mock_async_client.scenarios.scorers.create = AsyncMock(return_value=scorer_view) - client = AsyncScorerOps(mock_async_client) - scorer = await client.create( + ops = AsyncScorerOps(mock_async_client) + scorer = await ops.create( bash_script="echo 'score=1.0'", type="test_scorer", ) @@ -537,8 +537,8 @@ async def test_create(self, mock_async_client: AsyncMock, scorer_view: MockScore def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" - client = AsyncScorerOps(mock_async_client) - scorer = client.from_id("scorer_123") + ops = AsyncScorerOps(mock_async_client) + scorer = ops.from_id("scorer_123") assert isinstance(scorer, AsyncScorer) assert scorer.id == "scorer_123" @@ -552,8 +552,8 @@ async def async_iter(): mock_async_client.scenarios.scorers.list = AsyncMock(return_value=async_iter()) - client = AsyncScorerOps(mock_async_client) - scorers = await client.list( + ops = AsyncScorerOps(mock_async_client) + scorers = await ops.list( limit=10, starting_after="scorer_000", ) diff --git a/tests/sdk/test_clients.py b/tests/sdk/test_clients.py index ba6655eef..bae8be66e 100644 --- a/tests/sdk/test_clients.py +++ b/tests/sdk/test_clients.py @@ -35,8 +35,8 @@ def test_create(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: """Test create method.""" mock_client.devboxes.create_and_await_running.return_value = devbox_view - client = DevboxOps(mock_client) - devbox = client.create( + ops = DevboxOps(mock_client) + devbox = ops.create( name="test-devbox", metadata={"key": "value"}, polling_config=PollingConfig(timeout_seconds=60.0), @@ -50,8 +50,8 @@ def test_create_from_blueprint_id(self, mock_client: Mock, devbox_view: MockDevb """Test create_from_blueprint_id method.""" mock_client.devboxes.create_and_await_running.return_value = devbox_view - client = DevboxOps(mock_client) - devbox = client.create_from_blueprint_id( + ops = DevboxOps(mock_client) + devbox = ops.create_from_blueprint_id( "bp_123", name="test-devbox", metadata={"key": "value"}, @@ -66,8 +66,8 @@ def test_create_from_blueprint_name(self, mock_client: Mock, devbox_view: MockDe """Test create_from_blueprint_name method.""" mock_client.devboxes.create_and_await_running.return_value = devbox_view - client = DevboxOps(mock_client) - devbox = client.create_from_blueprint_name( + ops = DevboxOps(mock_client) + devbox = ops.create_from_blueprint_name( "my-blueprint", name="test-devbox", ) @@ -80,8 +80,8 @@ def test_create_from_snapshot(self, mock_client: Mock, devbox_view: MockDevboxVi """Test create_from_snapshot method.""" mock_client.devboxes.create_and_await_running.return_value = devbox_view - client = DevboxOps(mock_client) - devbox = client.create_from_snapshot( + ops = DevboxOps(mock_client) + devbox = ops.create_from_snapshot( "snap_123", name="test-devbox", ) @@ -94,8 +94,8 @@ def test_from_id(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: """Test from_id method waits for running.""" mock_client.devboxes.await_running.return_value = devbox_view - client = DevboxOps(mock_client) - devbox = client.from_id("dev_123") + ops = DevboxOps(mock_client) + devbox = ops.from_id("dev_123") assert isinstance(devbox, Devbox) assert devbox.id == "dev_123" @@ -106,8 +106,8 @@ def test_list(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: page = SimpleNamespace(devboxes=[devbox_view]) mock_client.devboxes.list.return_value = page - client = DevboxOps(mock_client) - devboxes = client.list( + ops = DevboxOps(mock_client) + devboxes = ops.list( limit=10, status="running", starting_after="dev_000", @@ -127,8 +127,8 @@ def test_list(self, mock_client: Mock, snapshot_view: MockSnapshotView) -> None: page = SimpleNamespace(snapshots=[snapshot_view]) mock_client.devboxes.disk_snapshots.list.return_value = page - client = SnapshotOps(mock_client) - snapshots = client.list( + ops = SnapshotOps(mock_client) + snapshots = ops.list( devbox_id="dev_123", limit=10, starting_after="snap_000", @@ -141,8 +141,8 @@ def test_list(self, mock_client: Mock, snapshot_view: MockSnapshotView) -> None: def test_from_id(self, mock_client: Mock) -> None: """Test from_id method.""" - client = SnapshotOps(mock_client) - snapshot = client.from_id("snap_123") + ops = SnapshotOps(mock_client) + snapshot = ops.from_id("snap_123") assert isinstance(snapshot, Snapshot) assert snapshot.id == "snap_123" @@ -155,8 +155,8 @@ def test_create(self, mock_client: Mock, blueprint_view: MockBlueprintView) -> N """Test create method.""" mock_client.blueprints.create_and_await_build_complete.return_value = blueprint_view - client = BlueprintOps(mock_client) - blueprint = client.create( + ops = BlueprintOps(mock_client) + blueprint = ops.create( name="test-blueprint", polling_config=PollingConfig(timeout_seconds=60.0), ) @@ -167,8 +167,8 @@ def test_create(self, mock_client: Mock, blueprint_view: MockBlueprintView) -> N def test_from_id(self, mock_client: Mock) -> None: """Test from_id method.""" - client = BlueprintOps(mock_client) - blueprint = client.from_id("bp_123") + ops = BlueprintOps(mock_client) + blueprint = ops.from_id("bp_123") assert isinstance(blueprint, Blueprint) assert blueprint.id == "bp_123" @@ -178,8 +178,8 @@ def test_list(self, mock_client: Mock, blueprint_view: MockBlueprintView) -> Non page = SimpleNamespace(blueprints=[blueprint_view]) mock_client.blueprints.list.return_value = page - client = BlueprintOps(mock_client) - blueprints = client.list( + ops = BlueprintOps(mock_client) + blueprints = ops.list( limit=10, name="test", starting_after="bp_000", @@ -198,8 +198,8 @@ def test_create(self, mock_client: Mock, object_view: MockObjectView) -> None: """Test create method.""" mock_client.objects.create.return_value = object_view - client = StorageObjectOps(mock_client) - obj = client.create(name="test.txt", content_type="text", metadata={"key": "value"}) + ops = StorageObjectOps(mock_client) + obj = ops.create(name="test.txt", content_type="text", metadata={"key": "value"}) assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -212,8 +212,8 @@ def test_create(self, mock_client: Mock, object_view: MockObjectView) -> None: def test_from_id(self, mock_client: Mock) -> None: """Test from_id method.""" - client = StorageObjectOps(mock_client) - obj = client.from_id("obj_123") + ops = StorageObjectOps(mock_client) + obj = ops.from_id("obj_123") assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -224,8 +224,8 @@ def test_list(self, mock_client: Mock, object_view: MockObjectView) -> None: page = SimpleNamespace(objects=[object_view]) mock_client.objects.list.return_value = page - client = StorageObjectOps(mock_client) - objects = client.list( + ops = StorageObjectOps(mock_client) + objects = ops.list( content_type="text", limit=10, name="test", @@ -258,8 +258,8 @@ def test_upload_from_file(self, mock_client: Mock, object_view: MockObjectView, http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) - obj = client.upload_from_file(temp_file, name="test.txt") + ops = StorageObjectOps(mock_client) + obj = ops.upload_from_file(temp_file, name="test.txt") assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -281,8 +281,8 @@ def test_upload_from_text(self, mock_client: Mock, object_view: MockObjectView) http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) - obj = client.upload_from_text("test content", name="test.txt", metadata={"key": "value"}) + ops = StorageObjectOps(mock_client) + obj = ops.upload_from_text("test content", name="test.txt", metadata={"key": "value"}) assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -304,8 +304,8 @@ def test_upload_from_bytes(self, mock_client: Mock, object_view: MockObjectView) http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) - obj = client.upload_from_bytes(b"test content", name="test.bin", content_type="binary") + ops = StorageObjectOps(mock_client) + obj = ops.upload_from_bytes(b"test content", name="test.bin", content_type="binary") assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -320,11 +320,11 @@ def test_upload_from_bytes(self, mock_client: Mock, object_view: MockObjectView) def test_upload_from_file_missing_path(self, mock_client: Mock, tmp_path: Path) -> None: """upload_from_file should raise when file cannot be read.""" - client = StorageObjectOps(mock_client) + ops = StorageObjectOps(mock_client) missing_file = tmp_path / "missing.txt" with pytest.raises(OSError, match="Failed to read file"): - client.upload_from_file(missing_file) + ops.upload_from_file(missing_file) def test_upload_from_dir(self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path) -> None: """Test upload_from_dir method.""" @@ -344,8 +344,8 @@ def test_upload_from_dir(self, mock_client: Mock, object_view: MockObjectView, t http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) - obj = client.upload_from_dir(test_dir, name="archive.tar.gz", metadata={"key": "value"}) + ops = StorageObjectOps(mock_client) + obj = ops.upload_from_dir(test_dir, name="archive.tar.gz", metadata={"key": "value"}) assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -377,8 +377,8 @@ def test_upload_from_dir_default_name(self, mock_client: Mock, object_view: Mock http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) - obj = client.upload_from_dir(test_dir) + ops = StorageObjectOps(mock_client) + obj = ops.upload_from_dir(test_dir) assert isinstance(obj, StorageObject) # Name should be directory name + .tar.gz @@ -404,8 +404,8 @@ def test_upload_from_dir_with_ttl(self, mock_client: Mock, object_view: MockObje http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) - obj = client.upload_from_dir(test_dir, ttl=timedelta(hours=2)) + ops = StorageObjectOps(mock_client) + obj = ops.upload_from_dir(test_dir, ttl=timedelta(hours=2)) assert isinstance(obj, StorageObject) mock_client.objects.create.assert_called_once_with( @@ -429,8 +429,8 @@ def test_upload_from_dir_empty_directory( http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) - obj = client.upload_from_dir(test_dir) + ops = StorageObjectOps(mock_client) + obj = ops.upload_from_dir(test_dir) assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -458,9 +458,9 @@ def test_upload_from_dir_with_string_path( http_client.put.return_value = mock_response mock_client._client = http_client - client = StorageObjectOps(mock_client) + ops = StorageObjectOps(mock_client) # Pass string path instead of Path object - obj = client.upload_from_dir(str(test_dir)) + obj = ops.upload_from_dir(str(test_dir)) assert isinstance(obj, StorageObject) assert obj.id == "obj_123" @@ -481,8 +481,8 @@ def test_create(self, mock_client: Mock, scorer_view: MockScorerView) -> None: """Test create method.""" mock_client.scenarios.scorers.create.return_value = scorer_view - client = ScorerOps(mock_client) - scorer = client.create( + ops = ScorerOps(mock_client) + scorer = ops.create( bash_script="echo 'score=1.0'", type="test_scorer", ) @@ -493,8 +493,8 @@ def test_create(self, mock_client: Mock, scorer_view: MockScorerView) -> None: def test_from_id(self, mock_client: Mock) -> None: """Test from_id method.""" - client = ScorerOps(mock_client) - scorer = client.from_id("scorer_123") + ops = ScorerOps(mock_client) + scorer = ops.from_id("scorer_123") assert isinstance(scorer, Scorer) assert scorer.id == "scorer_123" @@ -503,8 +503,8 @@ def test_list(self, mock_client: Mock, scorer_view: MockScorerView) -> None: """Test list method.""" mock_client.scenarios.scorers.list.return_value = [scorer_view] - client = ScorerOps(mock_client) - scorers = client.list( + ops = ScorerOps(mock_client) + scorers = ops.list( limit=10, starting_after="scorer_000", ) From e37d9e1c544a6599a62baedf9f4d8c588f08871f Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 18:52:04 -0500 Subject: [PATCH 10/13] rename client test file to ops --- tests/sdk/{test_async_clients.py => test_async_ops.py} | 0 tests/sdk/{test_clients.py => test_ops.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/sdk/{test_async_clients.py => test_async_ops.py} (100%) rename tests/sdk/{test_clients.py => test_ops.py} (100%) diff --git a/tests/sdk/test_async_clients.py b/tests/sdk/test_async_ops.py similarity index 100% rename from tests/sdk/test_async_clients.py rename to tests/sdk/test_async_ops.py diff --git a/tests/sdk/test_clients.py b/tests/sdk/test_ops.py similarity index 100% rename from tests/sdk/test_clients.py rename to tests/sdk/test_ops.py From bf11fd23956d0a905872fa46ef99a4a6b62df3d6 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 19:16:42 -0500 Subject: [PATCH 11/13] added list_empty, list_single and list_multiple unit tests to all ops class tests --- tests/sdk/test_async_ops.py | 198 ++++++++++++++++++++++++++++++++---- tests/sdk/test_ops.py | 178 ++++++++++++++++++++++++++++---- 2 files changed, 336 insertions(+), 40 deletions(-) diff --git a/tests/sdk/test_async_ops.py b/tests/sdk/test_async_ops.py index fd57a253e..e540f960a 100644 --- a/tests/sdk/test_async_ops.py +++ b/tests/sdk/test_async_ops.py @@ -30,8 +30,8 @@ from runloop_api_client.lib.polling import PollingConfig -class TestAsyncDevboxClient: - """Tests for AsyncDevboxClient class.""" +class TestAsyncDevboxOps: + """Tests for AsyncDevboxOps class.""" @pytest.mark.asyncio async def test_create(self, mock_async_client: AsyncMock, devbox_view: MockDevboxView) -> None: @@ -106,8 +106,20 @@ def test_from_id(self, mock_async_client: AsyncMock) -> None: assert not mock_async_client.devboxes.await_running.called @pytest.mark.asyncio - async def test_list(self, mock_async_client: AsyncMock, devbox_view: MockDevboxView) -> None: - """Test list method.""" + async def test_list_empty(self, mock_async_client: AsyncMock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(devboxes=[]) + mock_async_client.devboxes.list = AsyncMock(return_value=page) + + ops = AsyncDevboxOps(mock_async_client) + devboxes = await ops.list(limit=10, status="running") + + assert len(devboxes) == 0 + mock_async_client.devboxes.list.assert_called_once() + + @pytest.mark.asyncio + async def test_list_single(self, mock_async_client: AsyncMock, devbox_view: MockDevboxView) -> None: + """Test list method with single result.""" page = SimpleNamespace(devboxes=[devbox_view]) mock_async_client.devboxes.list = AsyncMock(return_value=page) @@ -123,13 +135,43 @@ async def test_list(self, mock_async_client: AsyncMock, devbox_view: MockDevboxV assert devboxes[0].id == "dev_123" mock_async_client.devboxes.list.assert_called_once() + @pytest.mark.asyncio + async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: + """Test list method with multiple results.""" + devbox_view1 = MockDevboxView(id="dev_001", name="devbox-1") + devbox_view2 = MockDevboxView(id="dev_002", name="devbox-2") + page = SimpleNamespace(devboxes=[devbox_view1, devbox_view2]) + mock_async_client.devboxes.list = AsyncMock(return_value=page) + + ops = AsyncDevboxOps(mock_async_client) + devboxes = await ops.list(limit=10, status="running") + + assert len(devboxes) == 2 + assert isinstance(devboxes[0], AsyncDevbox) + assert isinstance(devboxes[1], AsyncDevbox) + assert devboxes[0].id == "dev_001" + assert devboxes[1].id == "dev_002" + mock_async_client.devboxes.list.assert_called_once() + + +class TestAsyncSnapshotOps: + """Tests for AsyncSnapshotOps class.""" + + @pytest.mark.asyncio + async def test_list_empty(self, mock_async_client: AsyncMock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(snapshots=[]) + mock_async_client.devboxes.disk_snapshots.list = AsyncMock(return_value=page) + + ops = AsyncSnapshotOps(mock_async_client) + snapshots = await ops.list(devbox_id="dev_123", limit=10) -class TestAsyncSnapshotClient: - """Tests for AsyncSnapshotClient class.""" + assert len(snapshots) == 0 + mock_async_client.devboxes.disk_snapshots.list.assert_called_once() @pytest.mark.asyncio - async def test_list(self, mock_async_client: AsyncMock, snapshot_view: MockSnapshotView) -> None: - """Test list method.""" + async def test_list_single(self, mock_async_client: AsyncMock, snapshot_view: MockSnapshotView) -> None: + """Test list method with single result.""" page = SimpleNamespace(snapshots=[snapshot_view]) mock_async_client.devboxes.disk_snapshots.list = AsyncMock(return_value=page) @@ -145,6 +187,24 @@ async def test_list(self, mock_async_client: AsyncMock, snapshot_view: MockSnaps assert snapshots[0].id == "snap_123" mock_async_client.devboxes.disk_snapshots.list.assert_called_once() + @pytest.mark.asyncio + async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: + """Test list method with multiple results.""" + snapshot_view1 = MockSnapshotView(id="snap_001", name="snapshot-1") + snapshot_view2 = MockSnapshotView(id="snap_002", name="snapshot-2") + page = SimpleNamespace(snapshots=[snapshot_view1, snapshot_view2]) + mock_async_client.devboxes.disk_snapshots.list = AsyncMock(return_value=page) + + ops = AsyncSnapshotOps(mock_async_client) + snapshots = await ops.list(devbox_id="dev_123", limit=10) + + assert len(snapshots) == 2 + assert isinstance(snapshots[0], AsyncSnapshot) + assert isinstance(snapshots[1], AsyncSnapshot) + assert snapshots[0].id == "snap_001" + assert snapshots[1].id == "snap_002" + mock_async_client.devboxes.disk_snapshots.list.assert_called_once() + def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" ops = AsyncSnapshotOps(mock_async_client) @@ -154,8 +214,8 @@ def test_from_id(self, mock_async_client: AsyncMock) -> None: assert snapshot.id == "snap_123" -class TestAsyncBlueprintClient: - """Tests for AsyncBlueprintClient class.""" +class TestAsyncBlueprintOps: + """Tests for AsyncBlueprintOps class.""" @pytest.mark.asyncio async def test_create(self, mock_async_client: AsyncMock, blueprint_view: MockBlueprintView) -> None: @@ -181,8 +241,20 @@ def test_from_id(self, mock_async_client: AsyncMock) -> None: assert blueprint.id == "bp_123" @pytest.mark.asyncio - async def test_list(self, mock_async_client: AsyncMock, blueprint_view: MockBlueprintView) -> None: - """Test list method.""" + async def test_list_empty(self, mock_async_client: AsyncMock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(blueprints=[]) + mock_async_client.blueprints.list = AsyncMock(return_value=page) + + ops = AsyncBlueprintOps(mock_async_client) + blueprints = await ops.list(limit=10) + + assert len(blueprints) == 0 + mock_async_client.blueprints.list.assert_called_once() + + @pytest.mark.asyncio + async def test_list_single(self, mock_async_client: AsyncMock, blueprint_view: MockBlueprintView) -> None: + """Test list method with single result.""" page = SimpleNamespace(blueprints=[blueprint_view]) mock_async_client.blueprints.list = AsyncMock(return_value=page) @@ -198,9 +270,27 @@ async def test_list(self, mock_async_client: AsyncMock, blueprint_view: MockBlue assert blueprints[0].id == "bp_123" mock_async_client.blueprints.list.assert_called_once() + @pytest.mark.asyncio + async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: + """Test list method with multiple results.""" + blueprint_view1 = MockBlueprintView(id="bp_001", name="blueprint-1") + blueprint_view2 = MockBlueprintView(id="bp_002", name="blueprint-2") + page = SimpleNamespace(blueprints=[blueprint_view1, blueprint_view2]) + mock_async_client.blueprints.list = AsyncMock(return_value=page) -class TestAsyncStorageObjectClient: - """Tests for AsyncStorageObjectClient class.""" + ops = AsyncBlueprintOps(mock_async_client) + blueprints = await ops.list(limit=10) + + assert len(blueprints) == 2 + assert isinstance(blueprints[0], AsyncBlueprint) + assert isinstance(blueprints[1], AsyncBlueprint) + assert blueprints[0].id == "bp_001" + assert blueprints[1].id == "bp_002" + mock_async_client.blueprints.list.assert_called_once() + + +class TestAsyncStorageObjectOps: + """Tests for AsyncStorageObjectOps class.""" @pytest.mark.asyncio async def test_create(self, mock_async_client: AsyncMock, object_view: MockObjectView) -> None: @@ -229,8 +319,20 @@ def test_from_id(self, mock_async_client: AsyncMock) -> None: assert obj.upload_url is None @pytest.mark.asyncio - async def test_list(self, mock_async_client: AsyncMock, object_view: MockObjectView) -> None: - """Test list method.""" + async def test_list_empty(self, mock_async_client: AsyncMock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(objects=[]) + mock_async_client.objects.list = AsyncMock(return_value=page) + + ops = AsyncStorageObjectOps(mock_async_client) + objects = await ops.list(limit=10) + + assert len(objects) == 0 + mock_async_client.objects.list.assert_called_once() + + @pytest.mark.asyncio + async def test_list_single(self, mock_async_client: AsyncMock, object_view: MockObjectView) -> None: + """Test list method with single result.""" page = SimpleNamespace(objects=[object_view]) mock_async_client.objects.list = AsyncMock(return_value=page) @@ -256,6 +358,24 @@ async def test_list(self, mock_async_client: AsyncMock, object_view: MockObjectV state="READ_ONLY", ) + @pytest.mark.asyncio + async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: + """Test list method with multiple results.""" + object_view1 = MockObjectView(id="obj_001", name="object-1") + object_view2 = MockObjectView(id="obj_002", name="object-2") + page = SimpleNamespace(objects=[object_view1, object_view2]) + mock_async_client.objects.list = AsyncMock(return_value=page) + + ops = AsyncStorageObjectOps(mock_async_client) + objects = await ops.list(limit=10) + + assert len(objects) == 2 + assert isinstance(objects[0], AsyncStorageObject) + assert isinstance(objects[1], AsyncStorageObject) + assert objects[0].id == "obj_001" + assert objects[1].id == "obj_002" + mock_async_client.objects.list.assert_called_once() + @pytest.mark.asyncio async def test_upload_from_file( self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path @@ -517,8 +637,8 @@ async def test_upload_from_dir_with_string_path( mock_async_client.objects.complete.assert_awaited_once() -class TestAsyncScorerClient: - """Tests for AsyncScorerClient class.""" +class TestAsyncScorerOps: + """Tests for AsyncScorerOps class.""" @pytest.mark.asyncio async def test_create(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: @@ -544,8 +664,24 @@ def test_from_id(self, mock_async_client: AsyncMock) -> None: assert scorer.id == "scorer_123" @pytest.mark.asyncio - async def test_list(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: - """Test list method.""" + async def test_list_empty(self, mock_async_client: AsyncMock) -> None: + """Test list method with empty results.""" + + async def async_iter(): + return + yield # Make this a generator + + mock_async_client.scenarios.scorers.list = AsyncMock(return_value=async_iter()) + + ops = AsyncScorerOps(mock_async_client) + scorers = await ops.list(limit=10) + + assert len(scorers) == 0 + mock_async_client.scenarios.scorers.list.assert_called_once() + + @pytest.mark.asyncio + async def test_list_single(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: + """Test list method with single result.""" async def async_iter(): yield scorer_view @@ -563,6 +699,28 @@ async def async_iter(): assert scorers[0].id == "scorer_123" mock_async_client.scenarios.scorers.list.assert_called_once() + @pytest.mark.asyncio + async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: + """Test list method with multiple results.""" + scorer_view1 = MockScorerView(id="scorer_001", type="scorer-1") + scorer_view2 = MockScorerView(id="scorer_002", type="scorer-2") + + async def async_iter(): + yield scorer_view1 + yield scorer_view2 + + mock_async_client.scenarios.scorers.list = AsyncMock(return_value=async_iter()) + + ops = AsyncScorerOps(mock_async_client) + scorers = await ops.list(limit=10) + + assert len(scorers) == 2 + assert isinstance(scorers[0], AsyncScorer) + assert isinstance(scorers[1], AsyncScorer) + assert scorers[0].id == "scorer_001" + assert scorers[1].id == "scorer_002" + mock_async_client.scenarios.scorers.list.assert_called_once() + class TestAsyncRunloopSDK: """Tests for AsyncRunloopSDK class.""" diff --git a/tests/sdk/test_ops.py b/tests/sdk/test_ops.py index bae8be66e..83c9117f4 100644 --- a/tests/sdk/test_ops.py +++ b/tests/sdk/test_ops.py @@ -28,8 +28,8 @@ from runloop_api_client.lib.polling import PollingConfig -class TestDevboxClient: - """Tests for DevboxClient class.""" +class TestDevboxOps: + """Tests for DevboxOps class.""" def test_create(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: """Test create method.""" @@ -101,8 +101,19 @@ def test_from_id(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: assert devbox.id == "dev_123" mock_client.devboxes.await_running.assert_called_once_with("dev_123") - def test_list(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: - """Test list method.""" + def test_list_empty(self, mock_client: Mock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(devboxes=[]) + mock_client.devboxes.list.return_value = page + + ops = DevboxOps(mock_client) + devboxes = ops.list(limit=10, status="running") + + assert len(devboxes) == 0 + mock_client.devboxes.list.assert_called_once() + + def test_list_single(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: + """Test list method with single result.""" page = SimpleNamespace(devboxes=[devbox_view]) mock_client.devboxes.list.return_value = page @@ -118,12 +129,40 @@ def test_list(self, mock_client: Mock, devbox_view: MockDevboxView) -> None: assert devboxes[0].id == "dev_123" mock_client.devboxes.list.assert_called_once() + def test_list_multiple(self, mock_client: Mock) -> None: + """Test list method with multiple results.""" + devbox_view1 = MockDevboxView(id="dev_001", name="devbox-1") + devbox_view2 = MockDevboxView(id="dev_002", name="devbox-2") + page = SimpleNamespace(devboxes=[devbox_view1, devbox_view2]) + mock_client.devboxes.list.return_value = page + + ops = DevboxOps(mock_client) + devboxes = ops.list(limit=10, status="running") + + assert len(devboxes) == 2 + assert isinstance(devboxes[0], Devbox) + assert isinstance(devboxes[1], Devbox) + assert devboxes[0].id == "dev_001" + assert devboxes[1].id == "dev_002" + mock_client.devboxes.list.assert_called_once() + + +class TestSnapshotOps: + """Tests for SnapshotOps class.""" -class TestSnapshotClient: - """Tests for SnapshotClient class.""" + def test_list_empty(self, mock_client: Mock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(snapshots=[]) + mock_client.devboxes.disk_snapshots.list.return_value = page - def test_list(self, mock_client: Mock, snapshot_view: MockSnapshotView) -> None: - """Test list method.""" + ops = SnapshotOps(mock_client) + snapshots = ops.list(devbox_id="dev_123", limit=10) + + assert len(snapshots) == 0 + mock_client.devboxes.disk_snapshots.list.assert_called_once() + + def test_list_single(self, mock_client: Mock, snapshot_view: MockSnapshotView) -> None: + """Test list method with single result.""" page = SimpleNamespace(snapshots=[snapshot_view]) mock_client.devboxes.disk_snapshots.list.return_value = page @@ -139,6 +178,23 @@ def test_list(self, mock_client: Mock, snapshot_view: MockSnapshotView) -> None: assert snapshots[0].id == "snap_123" mock_client.devboxes.disk_snapshots.list.assert_called_once() + def test_list_multiple(self, mock_client: Mock) -> None: + """Test list method with multiple results.""" + snapshot_view1 = MockSnapshotView(id="snap_001", name="snapshot-1") + snapshot_view2 = MockSnapshotView(id="snap_002", name="snapshot-2") + page = SimpleNamespace(snapshots=[snapshot_view1, snapshot_view2]) + mock_client.devboxes.disk_snapshots.list.return_value = page + + ops = SnapshotOps(mock_client) + snapshots = ops.list(devbox_id="dev_123", limit=10) + + assert len(snapshots) == 2 + assert isinstance(snapshots[0], Snapshot) + assert isinstance(snapshots[1], Snapshot) + assert snapshots[0].id == "snap_001" + assert snapshots[1].id == "snap_002" + mock_client.devboxes.disk_snapshots.list.assert_called_once() + def test_from_id(self, mock_client: Mock) -> None: """Test from_id method.""" ops = SnapshotOps(mock_client) @@ -148,8 +204,8 @@ def test_from_id(self, mock_client: Mock) -> None: assert snapshot.id == "snap_123" -class TestBlueprintClient: - """Tests for BlueprintClient class.""" +class TestBlueprintOps: + """Tests for BlueprintOps class.""" def test_create(self, mock_client: Mock, blueprint_view: MockBlueprintView) -> None: """Test create method.""" @@ -173,8 +229,19 @@ def test_from_id(self, mock_client: Mock) -> None: assert isinstance(blueprint, Blueprint) assert blueprint.id == "bp_123" - def test_list(self, mock_client: Mock, blueprint_view: MockBlueprintView) -> None: - """Test list method.""" + def test_list_empty(self, mock_client: Mock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(blueprints=[]) + mock_client.blueprints.list.return_value = page + + ops = BlueprintOps(mock_client) + blueprints = ops.list(limit=10) + + assert len(blueprints) == 0 + mock_client.blueprints.list.assert_called_once() + + def test_list_single(self, mock_client: Mock, blueprint_view: MockBlueprintView) -> None: + """Test list method with single result.""" page = SimpleNamespace(blueprints=[blueprint_view]) mock_client.blueprints.list.return_value = page @@ -190,9 +257,26 @@ def test_list(self, mock_client: Mock, blueprint_view: MockBlueprintView) -> Non assert blueprints[0].id == "bp_123" mock_client.blueprints.list.assert_called_once() + def test_list_multiple(self, mock_client: Mock) -> None: + """Test list method with multiple results.""" + blueprint_view1 = MockBlueprintView(id="bp_001", name="blueprint-1") + blueprint_view2 = MockBlueprintView(id="bp_002", name="blueprint-2") + page = SimpleNamespace(blueprints=[blueprint_view1, blueprint_view2]) + mock_client.blueprints.list.return_value = page + + ops = BlueprintOps(mock_client) + blueprints = ops.list(limit=10) + + assert len(blueprints) == 2 + assert isinstance(blueprints[0], Blueprint) + assert isinstance(blueprints[1], Blueprint) + assert blueprints[0].id == "bp_001" + assert blueprints[1].id == "bp_002" + mock_client.blueprints.list.assert_called_once() + -class TestStorageObjectClient: - """Tests for StorageObjectClient class.""" +class TestStorageObjectOps: + """Tests for StorageObjectOps class.""" def test_create(self, mock_client: Mock, object_view: MockObjectView) -> None: """Test create method.""" @@ -219,8 +303,19 @@ def test_from_id(self, mock_client: Mock) -> None: assert obj.id == "obj_123" assert obj.upload_url is None - def test_list(self, mock_client: Mock, object_view: MockObjectView) -> None: - """Test list method.""" + def test_list_empty(self, mock_client: Mock) -> None: + """Test list method with empty results.""" + page = SimpleNamespace(objects=[]) + mock_client.objects.list.return_value = page + + ops = StorageObjectOps(mock_client) + objects = ops.list(limit=10) + + assert len(objects) == 0 + mock_client.objects.list.assert_called_once() + + def test_list_single(self, mock_client: Mock, object_view: MockObjectView) -> None: + """Test list method with single result.""" page = SimpleNamespace(objects=[object_view]) mock_client.objects.list.return_value = page @@ -246,6 +341,23 @@ def test_list(self, mock_client: Mock, object_view: MockObjectView) -> None: state="READ_ONLY", ) + def test_list_multiple(self, mock_client: Mock) -> None: + """Test list method with multiple results.""" + object_view1 = MockObjectView(id="obj_001", name="object-1") + object_view2 = MockObjectView(id="obj_002", name="object-2") + page = SimpleNamespace(objects=[object_view1, object_view2]) + mock_client.objects.list.return_value = page + + ops = StorageObjectOps(mock_client) + objects = ops.list(limit=10) + + assert len(objects) == 2 + assert isinstance(objects[0], StorageObject) + assert isinstance(objects[1], StorageObject) + assert objects[0].id == "obj_001" + assert objects[1].id == "obj_002" + mock_client.objects.list.assert_called_once() + def test_upload_from_file(self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path) -> None: """Test upload_from_file method.""" mock_client.objects.create.return_value = object_view @@ -474,8 +586,8 @@ def test_upload_from_dir_with_string_path( mock_client.objects.complete.assert_called_once() -class TestScorerClient: - """Tests for ScorerClient class.""" +class TestScorerOps: + """Tests for ScorerOps class.""" def test_create(self, mock_client: Mock, scorer_view: MockScorerView) -> None: """Test create method.""" @@ -499,8 +611,18 @@ def test_from_id(self, mock_client: Mock) -> None: assert isinstance(scorer, Scorer) assert scorer.id == "scorer_123" - def test_list(self, mock_client: Mock, scorer_view: MockScorerView) -> None: - """Test list method.""" + def test_list_empty(self, mock_client: Mock) -> None: + """Test list method with empty results.""" + mock_client.scenarios.scorers.list.return_value = [] + + ops = ScorerOps(mock_client) + scorers = ops.list(limit=10) + + assert len(scorers) == 0 + mock_client.scenarios.scorers.list.assert_called_once() + + def test_list_single(self, mock_client: Mock, scorer_view: MockScorerView) -> None: + """Test list method with single result.""" mock_client.scenarios.scorers.list.return_value = [scorer_view] ops = ScorerOps(mock_client) @@ -514,6 +636,22 @@ def test_list(self, mock_client: Mock, scorer_view: MockScorerView) -> None: assert scorers[0].id == "scorer_123" mock_client.scenarios.scorers.list.assert_called_once() + def test_list_multiple(self, mock_client: Mock) -> None: + """Test list method with multiple results.""" + scorer_view1 = MockScorerView(id="scorer_001", type="scorer-1") + scorer_view2 = MockScorerView(id="scorer_002", type="scorer-2") + mock_client.scenarios.scorers.list.return_value = [scorer_view1, scorer_view2] + + ops = ScorerOps(mock_client) + scorers = ops.list(limit=10) + + assert len(scorers) == 2 + assert isinstance(scorers[0], Scorer) + assert isinstance(scorers[1], Scorer) + assert scorers[0].id == "scorer_001" + assert scorers[1].id == "scorer_002" + mock_client.scenarios.scorers.list.assert_called_once() + class TestRunloopSDK: """Tests for RunloopSDK class.""" From 92b6130b9330b2631e263608639c56456c2c79ad Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 19:17:13 -0500 Subject: [PATCH 12/13] fix assert_called to assert_awaited --- tests/sdk/test_async_blueprint.py | 8 +++--- tests/sdk/test_async_execution_result.py | 4 +-- tests/sdk/test_async_ops.py | 34 ++++++++++++------------ tests/sdk/test_async_scorer.py | 6 ++--- tests/sdk/test_async_snapshot.py | 10 +++---- tests/sdk/test_async_storage_object.py | 10 +++---- 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/sdk/test_async_blueprint.py b/tests/sdk/test_async_blueprint.py index 8f638c18f..75901a445 100644 --- a/tests/sdk/test_async_blueprint.py +++ b/tests/sdk/test_async_blueprint.py @@ -38,7 +38,7 @@ async def test_get_info(self, mock_async_client: AsyncMock, blueprint_view: Mock ) assert result == blueprint_view - mock_async_client.blueprints.retrieve.assert_called_once() + mock_async_client.blueprints.retrieve.assert_awaited_once() @pytest.mark.asyncio async def test_logs(self, mock_async_client: AsyncMock) -> None: @@ -55,7 +55,7 @@ async def test_logs(self, mock_async_client: AsyncMock) -> None: ) assert result == logs_view - mock_async_client.blueprints.logs.assert_called_once() + mock_async_client.blueprints.logs.assert_awaited_once() @pytest.mark.asyncio async def test_delete(self, mock_async_client: AsyncMock) -> None: @@ -71,7 +71,7 @@ async def test_delete(self, mock_async_client: AsyncMock) -> None: ) assert result is not None # Verify return value is propagated - mock_async_client.blueprints.delete.assert_called_once() + mock_async_client.blueprints.delete.assert_awaited_once() @pytest.mark.asyncio async def test_create_devbox(self, mock_async_client: AsyncMock, devbox_view: MockDevboxView) -> None: @@ -87,4 +87,4 @@ async def test_create_devbox(self, mock_async_client: AsyncMock, devbox_view: Mo ) assert devbox.id == "dev_123" - mock_async_client.devboxes.create_and_await_running.assert_called_once() + mock_async_client.devboxes.create_and_await_running.assert_awaited_once() diff --git a/tests/sdk/test_async_execution_result.py b/tests/sdk/test_async_execution_result.py index a4df5bac9..2a71da1c7 100644 --- a/tests/sdk/test_async_execution_result.py +++ b/tests/sdk/test_async_execution_result.py @@ -190,7 +190,7 @@ async def mock_iter(): # Should stream full output output = await result.stdout() assert output == "line1\nline2\nline3\n" - mock_async_client.devboxes.executions.stream_stdout_updates.assert_called_once_with( + mock_async_client.devboxes.executions.stream_stdout_updates.assert_awaited_once_with( "exec_123", devbox_id="dev_123" ) @@ -226,7 +226,7 @@ async def mock_iter(): # Should stream full output output = await result.stderr() assert output == "error1\nerror2\n" - mock_async_client.devboxes.executions.stream_stderr_updates.assert_called_once_with( + mock_async_client.devboxes.executions.stream_stderr_updates.assert_awaited_once_with( "exec_123", devbox_id="dev_123" ) diff --git a/tests/sdk/test_async_ops.py b/tests/sdk/test_async_ops.py index e540f960a..340d86647 100644 --- a/tests/sdk/test_async_ops.py +++ b/tests/sdk/test_async_ops.py @@ -47,7 +47,7 @@ async def test_create(self, mock_async_client: AsyncMock, devbox_view: MockDevbo assert isinstance(devbox, AsyncDevbox) assert devbox.id == "dev_123" - mock_async_client.devboxes.create_and_await_running.assert_called_once() + mock_async_client.devboxes.create_and_await_running.assert_awaited_once() @pytest.mark.asyncio async def test_create_from_blueprint_id(self, mock_async_client: AsyncMock, devbox_view: MockDevboxView) -> None: @@ -115,7 +115,7 @@ async def test_list_empty(self, mock_async_client: AsyncMock) -> None: devboxes = await ops.list(limit=10, status="running") assert len(devboxes) == 0 - mock_async_client.devboxes.list.assert_called_once() + mock_async_client.devboxes.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_single(self, mock_async_client: AsyncMock, devbox_view: MockDevboxView) -> None: @@ -133,7 +133,7 @@ async def test_list_single(self, mock_async_client: AsyncMock, devbox_view: Mock assert len(devboxes) == 1 assert isinstance(devboxes[0], AsyncDevbox) assert devboxes[0].id == "dev_123" - mock_async_client.devboxes.list.assert_called_once() + mock_async_client.devboxes.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: @@ -151,7 +151,7 @@ async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: assert isinstance(devboxes[1], AsyncDevbox) assert devboxes[0].id == "dev_001" assert devboxes[1].id == "dev_002" - mock_async_client.devboxes.list.assert_called_once() + mock_async_client.devboxes.list.assert_awaited_once() class TestAsyncSnapshotOps: @@ -167,7 +167,7 @@ async def test_list_empty(self, mock_async_client: AsyncMock) -> None: snapshots = await ops.list(devbox_id="dev_123", limit=10) assert len(snapshots) == 0 - mock_async_client.devboxes.disk_snapshots.list.assert_called_once() + mock_async_client.devboxes.disk_snapshots.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_single(self, mock_async_client: AsyncMock, snapshot_view: MockSnapshotView) -> None: @@ -185,7 +185,7 @@ async def test_list_single(self, mock_async_client: AsyncMock, snapshot_view: Mo assert len(snapshots) == 1 assert isinstance(snapshots[0], AsyncSnapshot) assert snapshots[0].id == "snap_123" - mock_async_client.devboxes.disk_snapshots.list.assert_called_once() + mock_async_client.devboxes.disk_snapshots.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: @@ -203,7 +203,7 @@ async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: assert isinstance(snapshots[1], AsyncSnapshot) assert snapshots[0].id == "snap_001" assert snapshots[1].id == "snap_002" - mock_async_client.devboxes.disk_snapshots.list.assert_called_once() + mock_async_client.devboxes.disk_snapshots.list.assert_awaited_once() def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" @@ -230,7 +230,7 @@ async def test_create(self, mock_async_client: AsyncMock, blueprint_view: MockBl assert isinstance(blueprint, AsyncBlueprint) assert blueprint.id == "bp_123" - mock_async_client.blueprints.create_and_await_build_complete.assert_called_once() + mock_async_client.blueprints.create_and_await_build_complete.assert_awaited_once() def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" @@ -250,7 +250,7 @@ async def test_list_empty(self, mock_async_client: AsyncMock) -> None: blueprints = await ops.list(limit=10) assert len(blueprints) == 0 - mock_async_client.blueprints.list.assert_called_once() + mock_async_client.blueprints.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_single(self, mock_async_client: AsyncMock, blueprint_view: MockBlueprintView) -> None: @@ -268,7 +268,7 @@ async def test_list_single(self, mock_async_client: AsyncMock, blueprint_view: M assert len(blueprints) == 1 assert isinstance(blueprints[0], AsyncBlueprint) assert blueprints[0].id == "bp_123" - mock_async_client.blueprints.list.assert_called_once() + mock_async_client.blueprints.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: @@ -286,7 +286,7 @@ async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: assert isinstance(blueprints[1], AsyncBlueprint) assert blueprints[0].id == "bp_001" assert blueprints[1].id == "bp_002" - mock_async_client.blueprints.list.assert_called_once() + mock_async_client.blueprints.list.assert_awaited_once() class TestAsyncStorageObjectOps: @@ -328,7 +328,7 @@ async def test_list_empty(self, mock_async_client: AsyncMock) -> None: objects = await ops.list(limit=10) assert len(objects) == 0 - mock_async_client.objects.list.assert_called_once() + mock_async_client.objects.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_single(self, mock_async_client: AsyncMock, object_view: MockObjectView) -> None: @@ -374,7 +374,7 @@ async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: assert isinstance(objects[1], AsyncStorageObject) assert objects[0].id == "obj_001" assert objects[1].id == "obj_002" - mock_async_client.objects.list.assert_called_once() + mock_async_client.objects.list.assert_awaited_once() @pytest.mark.asyncio async def test_upload_from_file( @@ -653,7 +653,7 @@ async def test_create(self, mock_async_client: AsyncMock, scorer_view: MockScore assert isinstance(scorer, AsyncScorer) assert scorer.id == "scorer_123" - mock_async_client.scenarios.scorers.create.assert_called_once() + mock_async_client.scenarios.scorers.create.assert_awaited_once() def test_from_id(self, mock_async_client: AsyncMock) -> None: """Test from_id method.""" @@ -677,7 +677,7 @@ async def async_iter(): scorers = await ops.list(limit=10) assert len(scorers) == 0 - mock_async_client.scenarios.scorers.list.assert_called_once() + mock_async_client.scenarios.scorers.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_single(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: @@ -697,7 +697,7 @@ async def async_iter(): assert len(scorers) == 1 assert isinstance(scorers[0], AsyncScorer) assert scorers[0].id == "scorer_123" - mock_async_client.scenarios.scorers.list.assert_called_once() + mock_async_client.scenarios.scorers.list.assert_awaited_once() @pytest.mark.asyncio async def test_list_multiple(self, mock_async_client: AsyncMock) -> None: @@ -719,7 +719,7 @@ async def async_iter(): assert isinstance(scorers[1], AsyncScorer) assert scorers[0].id == "scorer_001" assert scorers[1].id == "scorer_002" - mock_async_client.scenarios.scorers.list.assert_called_once() + mock_async_client.scenarios.scorers.list.assert_awaited_once() class TestAsyncRunloopSDK: diff --git a/tests/sdk/test_async_scorer.py b/tests/sdk/test_async_scorer.py index 4cee049a3..7518498dd 100644 --- a/tests/sdk/test_async_scorer.py +++ b/tests/sdk/test_async_scorer.py @@ -38,7 +38,7 @@ async def test_get_info(self, mock_async_client: AsyncMock, scorer_view: MockSco result = await scorer.get_info() assert result == scorer_view - mock_async_client.scenarios.scorers.retrieve.assert_called_once() + mock_async_client.scenarios.scorers.retrieve.assert_awaited_once() @pytest.mark.asyncio async def test_update(self, mock_async_client: AsyncMock) -> None: @@ -53,7 +53,7 @@ async def test_update(self, mock_async_client: AsyncMock) -> None: ) assert result == update_response - mock_async_client.scenarios.scorers.update.assert_called_once() + mock_async_client.scenarios.scorers.update.assert_awaited_once() @pytest.mark.asyncio async def test_validate(self, mock_async_client: AsyncMock) -> None: @@ -71,4 +71,4 @@ async def test_validate(self, mock_async_client: AsyncMock) -> None: ) assert result == validate_response - mock_async_client.scenarios.scorers.validate.assert_called_once() + mock_async_client.scenarios.scorers.validate.assert_awaited_once() diff --git a/tests/sdk/test_async_snapshot.py b/tests/sdk/test_async_snapshot.py index 7bca2ad95..a7b946c11 100644 --- a/tests/sdk/test_async_snapshot.py +++ b/tests/sdk/test_async_snapshot.py @@ -39,7 +39,7 @@ async def test_get_info(self, mock_async_client: AsyncMock, snapshot_view: MockS ) assert result == snapshot_view - mock_async_client.devboxes.disk_snapshots.query_status.assert_called_once() + mock_async_client.devboxes.disk_snapshots.query_status.assert_awaited_once() @pytest.mark.asyncio async def test_update(self, mock_async_client: AsyncMock) -> None: @@ -60,7 +60,7 @@ async def test_update(self, mock_async_client: AsyncMock) -> None: ) assert result == updated_snapshot - mock_async_client.devboxes.disk_snapshots.update.assert_called_once() + mock_async_client.devboxes.disk_snapshots.update.assert_awaited_once() @pytest.mark.asyncio async def test_delete(self, mock_async_client: AsyncMock) -> None: @@ -77,7 +77,7 @@ async def test_delete(self, mock_async_client: AsyncMock) -> None: ) assert result is not None # Verify return value is propagated - mock_async_client.devboxes.disk_snapshots.delete.assert_called_once() + mock_async_client.devboxes.disk_snapshots.delete.assert_awaited_once() @pytest.mark.asyncio async def test_await_completed(self, mock_async_client: AsyncMock, snapshot_view: MockSnapshotView) -> None: @@ -95,7 +95,7 @@ async def test_await_completed(self, mock_async_client: AsyncMock, snapshot_view ) assert result == snapshot_view - mock_async_client.devboxes.disk_snapshots.await_completed.assert_called_once() + mock_async_client.devboxes.disk_snapshots.await_completed.assert_awaited_once() @pytest.mark.asyncio async def test_create_devbox(self, mock_async_client: AsyncMock, devbox_view: MockDevboxView) -> None: @@ -111,4 +111,4 @@ async def test_create_devbox(self, mock_async_client: AsyncMock, devbox_view: Mo ) assert devbox.id == "dev_123" - mock_async_client.devboxes.create_and_await_running.assert_called_once() + mock_async_client.devboxes.create_and_await_running.assert_awaited_once() diff --git a/tests/sdk/test_async_storage_object.py b/tests/sdk/test_async_storage_object.py index b4623a95a..434be5221 100644 --- a/tests/sdk/test_async_storage_object.py +++ b/tests/sdk/test_async_storage_object.py @@ -45,7 +45,7 @@ async def test_refresh(self, mock_async_client: AsyncMock, object_view: MockObje ) assert result == object_view - mock_async_client.objects.retrieve.assert_called_once() + mock_async_client.objects.retrieve.assert_awaited_once() @pytest.mark.asyncio async def test_complete(self, mock_async_client: AsyncMock) -> None: @@ -66,7 +66,7 @@ async def test_complete(self, mock_async_client: AsyncMock) -> None: assert result == completed_view assert obj.upload_url is None - mock_async_client.objects.complete.assert_called_once() + mock_async_client.objects.complete.assert_awaited_once() @pytest.mark.asyncio async def test_get_download_url_without_duration(self, mock_async_client: AsyncMock) -> None: @@ -83,7 +83,7 @@ async def test_get_download_url_without_duration(self, mock_async_client: AsyncM ) assert result == download_url_view - mock_async_client.objects.download.assert_called_once() + mock_async_client.objects.download.assert_awaited_once() @pytest.mark.asyncio async def test_get_download_url_with_duration(self, mock_async_client: AsyncMock) -> None: @@ -101,7 +101,7 @@ async def test_get_download_url_with_duration(self, mock_async_client: AsyncMock ) assert result == download_url_view - mock_async_client.objects.download.assert_called_once() + mock_async_client.objects.download.assert_awaited_once() @pytest.mark.asyncio async def test_download_as_bytes(self, mock_async_client: AsyncMock) -> None: @@ -160,7 +160,7 @@ async def test_delete(self, mock_async_client: AsyncMock, object_view: MockObjec ) assert result == object_view - mock_async_client.objects.delete.assert_called_once() + mock_async_client.objects.delete.assert_awaited_once() @pytest.mark.asyncio async def test_upload_content_string(self, mock_async_client: AsyncMock) -> None: From 7b68fa48765ce1d561ab0fc8f20e7b69c11980e3 Mon Sep 17 00:00:00 2001 From: Siddarth Chalasani Date: Mon, 1 Dec 2025 19:19:22 -0500 Subject: [PATCH 13/13] remove duplicate tests --- tests/sdk/test_async_scorer.py | 5 ----- tests/sdk/test_scorer.py | 5 ----- 2 files changed, 10 deletions(-) diff --git a/tests/sdk/test_async_scorer.py b/tests/sdk/test_async_scorer.py index 7518498dd..a3eeea884 100644 --- a/tests/sdk/test_async_scorer.py +++ b/tests/sdk/test_async_scorer.py @@ -24,11 +24,6 @@ def test_repr(self, mock_async_client: AsyncMock) -> None: scorer = AsyncScorer(mock_async_client, "scorer_123") assert repr(scorer) == "" - def test_id_property(self, mock_async_client: AsyncMock) -> None: - """Test id property returns the scorer ID.""" - scorer = AsyncScorer(mock_async_client, "scorer_123") - assert scorer.id == "scorer_123" - @pytest.mark.asyncio async def test_get_info(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None: """Test get_info method.""" diff --git a/tests/sdk/test_scorer.py b/tests/sdk/test_scorer.py index 6947e0f08..761a487cb 100644 --- a/tests/sdk/test_scorer.py +++ b/tests/sdk/test_scorer.py @@ -22,11 +22,6 @@ def test_repr(self, mock_client: Mock) -> None: scorer = Scorer(mock_client, "scorer_123") assert repr(scorer) == "" - def test_id_property(self, mock_client: Mock) -> None: - """Test id property returns the scorer ID.""" - scorer = Scorer(mock_client, "scorer_123") - assert scorer.id == "scorer_123" - def test_get_info(self, mock_client: Mock, scorer_view: MockScorerView) -> None: """Test get_info method.""" mock_client.scenarios.scorers.retrieve.return_value = scorer_view