scorer tests

Author: sid-rl

tests/sdk/conftest.py

@@ -20,6 +20,7 @@
     "snapshot": "snap_123",
     "blueprint": "bp_123",
     "object": "obj_123",
+    "scorer": "scorer_123",
 }
 
 # Test URL constants
@@ -86,6 +87,15 @@ class MockObjectView:
     name: str = "test-object"
 
 
+@dataclass
+class MockScorerView:
+    """Mock ScorerView for testing."""
+
+    id: str = "scorer_123"
+    bash_script: str = "echo 'score=1.0'"
+    type: str = "test_scorer"
+
+
 def create_mock_httpx_client(methods: dict[str, Any] | None = None) -> AsyncMock:
     """
     Create a mock httpx.AsyncClient with proper context manager setup.
@@ -170,6 +180,12 @@ def object_view() -> MockObjectView:
     return MockObjectView()
 
 
+@pytest.fixture
+def scorer_view() -> MockScorerView:
+    """Create a mock ScorerView."""
+    return MockScorerView()
+
+
 @pytest.fixture
 def mock_httpx_response() -> Mock:
     """Create a mock httpx.Response."""

tests/sdk/test_async_clients.py

@@ -13,13 +13,15 @@
 from tests.sdk.conftest import (
     MockDevboxView,
     MockObjectView,
+    MockScorerView,
     MockSnapshotView,
     MockBlueprintView,
     create_mock_httpx_response,
 )
-from runloop_api_client.sdk import AsyncDevbox, AsyncSnapshot, AsyncBlueprint, AsyncStorageObject
+from runloop_api_client.sdk import AsyncDevbox, AsyncScorer, AsyncSnapshot, AsyncBlueprint, AsyncStorageObject
 from runloop_api_client.sdk.async_ import (
     AsyncDevboxOps,
+    AsyncScorerOps,
     AsyncRunloopSDK,
     AsyncSnapshotOps,
     AsyncBlueprintOps,
@@ -515,6 +517,53 @@ async def test_upload_from_dir_with_string_path(
         mock_async_client.objects.complete.assert_awaited_once()
 
 
+class TestAsyncScorerClient:
+    """Tests for AsyncScorerClient class."""
+
+    @pytest.mark.asyncio
+    async def test_create(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None:
+        """Test create method."""
+        mock_async_client.scenarios.scorers.create = AsyncMock(return_value=scorer_view)
+
+        client = AsyncScorerOps(mock_async_client)
+        scorer = await client.create(
+            bash_script="echo 'score=1.0'",
+            type="test_scorer",
+        )
+
+        assert isinstance(scorer, AsyncScorer)
+        assert scorer.id == "scorer_123"
+        mock_async_client.scenarios.scorers.create.assert_called_once()
+
+    def test_from_id(self, mock_async_client: AsyncMock) -> None:
+        """Test from_id method."""
+        client = AsyncScorerOps(mock_async_client)
+        scorer = client.from_id("scorer_123")
+
+        assert isinstance(scorer, AsyncScorer)
+        assert scorer.id == "scorer_123"
+
+    @pytest.mark.asyncio
+    async def test_list(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None:
+        """Test list method."""
+
+        async def async_iter():
+            yield scorer_view
+
+        mock_async_client.scenarios.scorers.list = AsyncMock(return_value=async_iter())
+
+        client = AsyncScorerOps(mock_async_client)
+        scorers = await client.list(
+            limit=10,
+            starting_after="scorer_000",
+        )
+
+        assert len(scorers) == 1
+        assert isinstance(scorers[0], AsyncScorer)
+        assert scorers[0].id == "scorer_123"
+        mock_async_client.scenarios.scorers.list.assert_called_once()
+
+
 class TestAsyncRunloopSDK:
     """Tests for AsyncRunloopSDK class."""
 
@@ -523,6 +572,7 @@ def test_init(self) -> None:
         sdk = AsyncRunloopSDK(bearer_token="test-token")
         assert sdk.api is not None
         assert isinstance(sdk.devbox, AsyncDevboxOps)
+        assert isinstance(sdk.scorer, AsyncScorerOps)
         assert isinstance(sdk.snapshot, AsyncSnapshotOps)
         assert isinstance(sdk.blueprint, AsyncBlueprintOps)
         assert isinstance(sdk.storage_object, AsyncStorageObjectOps)

tests/sdk/test_async_scorer.py

@@ -0,0 +1,89 @@
+"""Comprehensive tests for async AsyncScorer class."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from tests.sdk.conftest import MockScorerView
+from runloop_api_client.sdk import AsyncScorer
+
+
+class TestAsyncScorer:
+    """Tests for AsyncScorer class."""
+
+    def test_init(self, mock_async_client: AsyncMock) -> None:
+        """Test AsyncScorer initialization."""
+        scorer = AsyncScorer(mock_async_client, "scorer_123")
+        assert scorer.id == "scorer_123"
+
+    def test_repr(self, mock_async_client: AsyncMock) -> None:
+        """Test AsyncScorer string representation."""
+        scorer = AsyncScorer(mock_async_client, "scorer_123")
+        assert repr(scorer) == "<AsyncScorer id='scorer_123'>"
+
+    def test_id_property(self, mock_async_client: AsyncMock) -> None:
+        """Test id property returns the scorer ID."""
+        scorer = AsyncScorer(mock_async_client, "scorer_123")
+        assert scorer.id == "scorer_123"
+
+    @pytest.mark.asyncio
+    async def test_get_info(self, mock_async_client: AsyncMock, scorer_view: MockScorerView) -> None:
+        """Test get_info method."""
+        mock_async_client.scenarios.scorers.retrieve = AsyncMock(return_value=scorer_view)
+
+        scorer = AsyncScorer(mock_async_client, "scorer_123")
+        result = await scorer.get_info(
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+        assert result == scorer_view
+        mock_async_client.scenarios.scorers.retrieve.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_update(self, mock_async_client: AsyncMock) -> None:
+        """Test update method."""
+        update_response = SimpleNamespace(id="scorer_123", name="updated-scorer")
+        mock_async_client.scenarios.scorers.update = AsyncMock(return_value=update_response)
+
+        scorer = AsyncScorer(mock_async_client, "scorer_123")
+        result = await scorer.update(
+            name="updated-scorer",
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+        assert result == update_response
+        mock_async_client.scenarios.scorers.update.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_validate(self, mock_async_client: AsyncMock) -> None:
+        """Test validate method."""
+        validate_response = SimpleNamespace(
+            is_valid=True,
+            score=0.95,
+            reasoning="The output matches expected criteria.",
+        )
+        mock_async_client.scenarios.scorers.validate = AsyncMock(return_value=validate_response)
+
+        scorer = AsyncScorer(mock_async_client, "scorer_123")
+        result = await scorer.validate(
+            bash_command_output="test output",
+            expected_output="test output",
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+        assert result == validate_response
+        assert result.is_valid is True
+        assert result.score == 0.95
+        mock_async_client.scenarios.scorers.validate.assert_called_once()

tests/sdk/test_clients.py

@@ -11,13 +11,15 @@
 from tests.sdk.conftest import (
     MockDevboxView,
     MockObjectView,
+    MockScorerView,
     MockSnapshotView,
     MockBlueprintView,
     create_mock_httpx_response,
 )
-from runloop_api_client.sdk import Devbox, Snapshot, Blueprint, StorageObject
+from runloop_api_client.sdk import Devbox, Scorer, Snapshot, Blueprint, StorageObject
 from runloop_api_client.sdk.sync import (
     DevboxOps,
+    ScorerOps,
     RunloopSDK,
     SnapshotOps,
     BlueprintOps,
@@ -472,6 +474,47 @@ def test_upload_from_dir_with_string_path(
         mock_client.objects.complete.assert_called_once()
 
 
+class TestScorerClient:
+    """Tests for ScorerClient class."""
+
+    def test_create(self, mock_client: Mock, scorer_view: MockScorerView) -> None:
+        """Test create method."""
+        mock_client.scenarios.scorers.create.return_value = scorer_view
+
+        client = ScorerOps(mock_client)
+        scorer = client.create(
+            bash_script="echo 'score=1.0'",
+            type="test_scorer",
+        )
+
+        assert isinstance(scorer, Scorer)
+        assert scorer.id == "scorer_123"
+        mock_client.scenarios.scorers.create.assert_called_once()
+
+    def test_from_id(self, mock_client: Mock) -> None:
+        """Test from_id method."""
+        client = ScorerOps(mock_client)
+        scorer = client.from_id("scorer_123")
+
+        assert isinstance(scorer, Scorer)
+        assert scorer.id == "scorer_123"
+
+    def test_list(self, mock_client: Mock, scorer_view: MockScorerView) -> None:
+        """Test list method."""
+        mock_client.scenarios.scorers.list.return_value = [scorer_view]
+
+        client = ScorerOps(mock_client)
+        scorers = client.list(
+            limit=10,
+            starting_after="scorer_000",
+        )
+
+        assert len(scorers) == 1
+        assert isinstance(scorers[0], Scorer)
+        assert scorers[0].id == "scorer_123"
+        mock_client.scenarios.scorers.list.assert_called_once()
+
+
 class TestRunloopSDK:
     """Tests for RunloopSDK class."""
 
@@ -480,6 +523,7 @@ def test_init(self) -> None:
         sdk = RunloopSDK(bearer_token="test-token")
         assert sdk.api is not None
         assert isinstance(sdk.devbox, DevboxOps)
+        assert isinstance(sdk.scorer, ScorerOps)
         assert isinstance(sdk.snapshot, SnapshotOps)
         assert isinstance(sdk.blueprint, BlueprintOps)
         assert isinstance(sdk.storage_object, StorageObjectOps)

tests/sdk/test_scorer.py

@@ -0,0 +1,105 @@
+"""Comprehensive tests for sync Scorer class."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+from tests.sdk.conftest import MockScorerView
+from runloop_api_client.sdk import Scorer
+
+
+class TestScorer:
+    """Tests for Scorer class."""
+
+    def test_init(self, mock_client: Mock) -> None:
+        """Test Scorer initialization."""
+        scorer = Scorer(mock_client, "scorer_123")
+        assert scorer.id == "scorer_123"
+
+    def test_repr(self, mock_client: Mock) -> None:
+        """Test Scorer string representation."""
+        scorer = Scorer(mock_client, "scorer_123")
+        assert repr(scorer) == "<Scorer id='scorer_123'>"
+
+    def test_id_property(self, mock_client: Mock) -> None:
+        """Test id property returns the scorer ID."""
+        scorer = Scorer(mock_client, "scorer_123")
+        assert scorer.id == "scorer_123"
+
+    def test_get_info(self, mock_client: Mock, scorer_view: MockScorerView) -> None:
+        """Test get_info method."""
+        mock_client.scenarios.scorers.retrieve.return_value = scorer_view
+
+        scorer = Scorer(mock_client, "scorer_123")
+        result = scorer.get_info(
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+        assert result == scorer_view
+        mock_client.scenarios.scorers.retrieve.assert_called_once_with(
+            "scorer_123",
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+    def test_update(self, mock_client: Mock) -> None:
+        """Test update method."""
+        update_response = SimpleNamespace(id="scorer_123", name="updated-scorer")
+        mock_client.scenarios.scorers.update.return_value = update_response
+
+        scorer = Scorer(mock_client, "scorer_123")
+        result = scorer.update(
+            name="updated-scorer",
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+        assert result == update_response
+        mock_client.scenarios.scorers.update.assert_called_once_with(
+            "scorer_123",
+            name="updated-scorer",
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+    def test_validate(self, mock_client: Mock) -> None:
+        """Test validate method."""
+        validate_response = SimpleNamespace(
+            is_valid=True,
+            score=0.95,
+            reasoning="The output matches expected criteria.",
+        )
+        mock_client.scenarios.scorers.validate.return_value = validate_response
+
+        scorer = Scorer(mock_client, "scorer_123")
+        result = scorer.validate(
+            bash_command_output="test output",
+            expected_output="test output",
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )
+
+        assert result == validate_response
+        assert result.is_valid is True
+        assert result.score == 0.95
+        mock_client.scenarios.scorers.validate.assert_called_once_with(
+            "scorer_123",
+            bash_command_output="test output",
+            expected_output="test output",
+            extra_headers={"X-Custom": "value"},
+            extra_query={"param": "value"},
+            extra_body={"key": "value"},
+            timeout=30.0,
+        )