From 010ab3c0f09756e5c407cafcdb055ab6256ba55c Mon Sep 17 00:00:00 2001 From: Guillaume Aquilina Date: Tue, 1 Oct 2024 14:15:58 -0400 Subject: [PATCH 1/2] test: add deserialization test --- poetry.lock | 12 +-- tests/fixtures/task_run_float_usage.json | 85 ++++++++++++++++++ workflowai/core/client/models_test.py | 17 ++++ workflowai/core/domain/task_run_test.py | 110 +++++++++++++++++++++++ 4 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/task_run_float_usage.json create mode 100644 workflowai/core/client/models_test.py create mode 100644 workflowai/core/domain/task_run_test.py diff --git a/poetry.lock b/poetry.lock index dc99e0e..4cc765a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -477,21 +477,23 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyright" -version = "1.1.381" +version = "1.1.383" description = "Command line wrapper for pyright" optional = false python-versions = ">=3.7" files = [ - {file = "pyright-1.1.381-py3-none-any.whl", hash = "sha256:5dc0aa80a265675d36abab59c674ae01dbe476714f91845b61b841d34aa99081"}, - {file = "pyright-1.1.381.tar.gz", hash = "sha256:314cf0c1351c189524fb10c7ac20688ecd470e8cc505c394d642c9c80bf7c3a5"}, + {file = "pyright-1.1.383-py3-none-any.whl", hash = "sha256:d864d1182a313f45aaf99e9bfc7d2668eeabc99b29a556b5344894fd73cb1959"}, + {file = "pyright-1.1.383.tar.gz", hash = "sha256:1df7f12407f3710c9c6df938d98ec53f70053e6c6bbf71ce7bcb038d42f10070"}, ] [package.dependencies] nodeenv = ">=1.6.0" +typing-extensions = ">=4.1" [package.extras] -all = ["twine (>=3.4.1)"] +all = ["nodejs-wheel-binaries", "twine (>=3.4.1)"] dev = ["twine (>=3.4.1)"] +nodejs = ["nodejs-wheel-binaries"] [[package]] name = "pytest" diff --git a/tests/fixtures/task_run_float_usage.json b/tests/fixtures/task_run_float_usage.json new file mode 100644 index 0000000..18d0899 --- /dev/null +++ b/tests/fixtures/task_run_float_usage.json @@ -0,0 +1,85 @@ +{ + "id": "e3683ced-efd2-4b15-9ab4-aefdf17c4c19", + "task_id": "generate-changelog-from-properties", + "task_schema_id": 1, + "task_input": { + "temperature": 1 + }, + "task_input_hash": "6faebdc55135f380b00e7ada53f5cccc", + "task_input_preview": "old_task_group: {properties: {temperature: 1, instructions: \"Add 5 to the n", + "task_output": { + "changes": [ + "Temperature decreased from Creative to 0.73" + ] + }, + "task_output_hash": "d057c1f26fd8447c11cda7300e0f3717", + "task_output_preview": "changes: [\"Temperature decreased from Creative to 0.73\"]", + "group": { + "id": "68eead780d01791ff2e09d39055ae6e8", + "iteration": 36, + "properties": { + "model": "gemini-1.5-flash-002", + "provider": "google", + "temperature": 0, + "instructions": "", + "max_tokens": null, + "runner_name": "WorkflowAI", + "runner_version": "v0.1.0", + "few_shot": null, + "template_name": "v1", + "task_variant_id": "fa546275ed8f6c801d6c6f174828d615" + }, + "tags": [ + "model=gemini-1.5-flash-002", + "provider=google", + "temperature=0" + ], + "aliases": null, + "is_external": null, + "is_favorite": null, + "notes": null, + "similarity_hash": "", + "benchmark_for_datasets": null + }, + "status": "success", + "error": null, + "start_time": "2024-10-01T17:55:06.241000Z", + "end_time": "2024-10-01T17:55:07.879000Z", + "duration_seconds": 1.638103, + "cost_usd": 0.00004651875, + "created_at": "2024-10-01T17:55:07.879000Z", + "updated_at": "2024-10-01T17:55:07.879000Z", + "example_id": null, + "corrections": null, + "parent_task_ids": null, + "scores": null, + "labels": null, + "metadata": { + "used_alias": "environment=production" + }, + "llm_completions": [ + { + "messages": [ + { + "role": "system", + "content": "" + }, + { + "role": "user", + "content": "" + } + ], + "response": "{\"changes\": [\"Temperature decreased from Creative to 0.73\"]}", + "usage": { + "completion_token_count": 13.5, + "completion_cost_usd": 0.00000405, + "prompt_token_count": 566.25, + "prompt_cost_usd": 0.00004246875 + } + } + ], + "config_id": null, + "dataset_benchmark_ids": null, + "is_free": null, + "author_tenant": null +} diff --git a/workflowai/core/client/models_test.py b/workflowai/core/client/models_test.py new file mode 100644 index 0000000..8038cb6 --- /dev/null +++ b/workflowai/core/client/models_test.py @@ -0,0 +1,17 @@ +import pytest + +from tests.utils import fixture_text +from workflowai.core.client.models import TaskRunResponse + + +@pytest.mark.parametrize( + "fixture", + [ + "task_run.json", + "task_run_float_usage.json", + ], +) +def test_task_run_response(fixture: str): + txt = fixture_text(fixture) + task_run = TaskRunResponse.model_validate_json(txt) + assert task_run diff --git a/workflowai/core/domain/task_run_test.py b/workflowai/core/domain/task_run_test.py new file mode 100644 index 0000000..079354d --- /dev/null +++ b/workflowai/core/domain/task_run_test.py @@ -0,0 +1,110 @@ +from typing import Optional + +from pydantic import BaseModel + +from workflowai.core.client.models import TaskRunResponse +from workflowai.core.domain.task import Task + + +class GenerateChangelogFromPropertiesTaskInput(BaseModel): + temperature: Optional[float] = None + + +class GenerateChangelogFromPropertiesTaskOutput(BaseModel): + changes: Optional[list[str]] = None + + +def test_task_run_model_validate(): + json_str = """{ + "id": "e3683ced-efd2-4b15-9ab4-aefdf17c4c19", + "task_id": "generate-changelog-from-properties", + "task_schema_id": 1, + "task_input": { + "temperature": 1 + }, + "task_input_hash": "6faebdc55135f380b00e7ada53f5cccc", + "task_input_preview": "old_task_group: {properties: {temperature: 1, instructions: \\"Add 5 to the n", + "task_output": { + "changes": [ + "Temperature decreased from Creative to 0.73" + ] + }, + "task_output_hash": "d057c1f26fd8447c11cda7300e0f3717", + "task_output_preview": "changes: [\\"Temperature decreased from Creative to 0.73\\"]", + "group": { + "id": "68eead780d01791ff2e09d39055ae6e8", + "iteration": 36, + "properties": { + "model": "gemini-1.5-flash-002", + "provider": "google", + "temperature": 0, + "instructions": "", + "max_tokens": null, + "runner_name": "WorkflowAI", + "runner_version": "v0.1.0", + "few_shot": null, + "template_name": "v1", + "task_variant_id": "fa546275ed8f6c801d6c6f174828d615" + }, + "tags": [ + "model=gemini-1.5-flash-002", + "provider=google", + "temperature=0" + ], + "aliases": null, + "is_external": null, + "is_favorite": null, + "notes": null, + "similarity_hash": "", + "benchmark_for_datasets": null + }, + "status": "success", + "error": null, + "start_time": "2024-10-01T17:55:06.241000Z", + "end_time": "2024-10-01T17:55:07.879000Z", + "duration_seconds": 1.638103, + "cost_usd": 0.00004651875, + "created_at": "2024-10-01T17:55:07.879000Z", + "updated_at": "2024-10-01T17:55:07.879000Z", + "example_id": null, + "corrections": null, + "parent_task_ids": null, + "scores": null, + "labels": null, + "metadata": { + "used_alias": "environment=production" + }, + "llm_completions": [ + { + "messages": [ + { + "role": "system", + "content": "" + }, + { + "role": "user", + "content": "" + } + ], + "response": "{\\"changes\\": [\\"Temperature decreased from Creative to 0.73\\"]}", + "usage": { + "completion_token_count": 13.5, + "completion_cost_usd": 0.00000405, + "prompt_token_count": 566.25, + "prompt_cost_usd": 0.00004246875 + } + } + ], + "config_id": null, + "dataset_benchmark_ids": null, + "is_free": null, + "author_tenant": null +}""" + + task_run = TaskRunResponse.model_validate_json(json_str).to_domain( + Task( + input_class=GenerateChangelogFromPropertiesTaskInput, + output_class=GenerateChangelogFromPropertiesTaskOutput, + ), + ) + assert task_run.id == "e3683ced-efd2-4b15-9ab4-aefdf17c4c19" From 3e1c45b52867cfabf9fa7d96f0dce33047d89f8a Mon Sep 17 00:00:00 2001 From: Guillaume Aquilina Date: Fri, 4 Oct 2024 14:50:16 -0400 Subject: [PATCH 2/2] fix: typing --- workflowai/core/client/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflowai/core/client/api.py b/workflowai/core/client/api.py index c3525bb..5c53fd5 100644 --- a/workflowai/core/client/api.py +++ b/workflowai/core/client/api.py @@ -22,11 +22,12 @@ def __init__(self, endpoint: str, api_key: str, source_headers: Optional[dict[st raise ValueError("Missing API URL or key") def _client(self) -> httpx.AsyncClient: + source_headers = self.source_headers or {} client = httpx.AsyncClient( base_url=self.endpoint, headers={ "Authorization": f"Bearer {self.api_key}", - **(self.source_headers or {}), + **source_headers, }, timeout=120.0, )