From d997b3eff6434835ce62d10a559348bb363f4123 Mon Sep 17 00:00:00 2001 From: Amy Ni Date: Fri, 29 Aug 2025 12:34:27 -0400 Subject: [PATCH 1/4] feat: initial access to modeling data via existing endpoints --- examples/get_modeling_data_example.py | 102 +++++++++ examples/quick_modeling_test.py | 82 ++++++++ examples/test_modeling_api_end_to_end.py | 254 +++++++++++++++++++++++ src/cvec/cvec.py | 59 ++++++ src/cvec/models/__init__.py | 24 ++- src/cvec/models/modeling.py | 55 +++++ tests/test_modeling.py | 209 +++++++++++++++++++ 7 files changed, 784 insertions(+), 1 deletion(-) create mode 100644 examples/get_modeling_data_example.py create mode 100644 examples/quick_modeling_test.py create mode 100644 examples/test_modeling_api_end_to_end.py create mode 100644 src/cvec/models/modeling.py create mode 100644 tests/test_modeling.py diff --git a/examples/get_modeling_data_example.py b/examples/get_modeling_data_example.py new file mode 100644 index 0000000..56642f3 --- /dev/null +++ b/examples/get_modeling_data_example.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Example script demonstrating how to retrieve modeling data from the CVec API. + +This script shows how to: +1. Fetch modeling readings for specific tag IDs within a time range +2. Get the latest modeling readings for specific tag IDs +3. Handle the response data + +Make sure to set the following environment variables: +- CVEC_HOST: Your CVec backend host URL +- CVEC_API_KEY: Your CVec API key +""" + +import os +from datetime import datetime, timedelta +from typing import List + +from cvec import CVec +from cvec.models.modeling import ModelingReadingsGroup, ModelingReadingModel + + +def print_modeling_readings(readings: List[ModelingReadingsGroup]) -> None: + """Print modeling readings in a formatted way.""" + print(f"Retrieved {len(readings)} tag groups:") + print("-" * 50) + + for group in readings: + print(f"Tag ID: {group.tag_id}") + print(f"Source: {group.source}") + print(f"Number of data points: {len(group.data)}") + + if group.data: + # Show first and last few data points + print(" First few data points:") + for i, point in enumerate(group.data[:3]): + timestamp = datetime.fromtimestamp(point.timestamp) + print(f" {i+1}. Time: {timestamp}, Value: {point.tag_value}") + + if len(group.data) > 3: + print(f" ... and {len(group.data) - 3} more points") + + # Show last data point + last_point = group.data[-1] + last_timestamp = datetime.fromtimestamp(last_point.timestamp) + print(f" Last data point: Time: {last_timestamp}, Value: {last_point.tag_value}") + + print("-" * 30) + + +def main(): + """Main function demonstrating modeling data retrieval.""" + try: + # Initialize CVec client + cvec = CVec() + print(f"Connected to CVec at: {cvec.host}") + + # Example tag IDs (replace with actual tag IDs from your system) + tag_ids = [1, 2, 3] + + # Set time range (last 24 hours) + end_date = datetime.now() + start_date = end_date - timedelta(hours=24) + + print(f"Fetching modeling data for tag IDs: {tag_ids}") + print(f"Time range: {start_date} to {end_date}") + print("=" * 60) + + # Fetch modeling readings + print("1. Fetching modeling readings...") + modeling_response = cvec.get_modeling_readings( + tag_ids=tag_ids, + start_at=start_date, + end_at=end_date, + desired_points=1000, # Limit to 1000 points for performance + ) + + print(f"Successfully retrieved modeling data with {len(modeling_response.items)} tag groups") + print_modeling_readings(modeling_response.items) + + print("\n" + "=" * 60) + + # Get latest modeling readings + print("2. Fetching latest modeling readings...") + latest_response = cvec.get_modeling_latest_readings(tag_ids=tag_ids) + + print(f"Successfully retrieved latest readings for {len(latest_response.items)} tags:") + for item in latest_response.items: + print(f" Tag {item.tag_id}: Value {item.tag_value} at {item.tag_value_changed_at}") + + print("\n" + "=" * 60) + print("Example completed successfully!") + + except Exception as e: + print(f"Error occurred: {e}") + print("Make sure your environment variables are set correctly:") + print("- CVEC_HOST: Your CVec backend host URL") + print("- CVEC_API_KEY: Your CVec API key") + + +if __name__ == "__main__": + main() diff --git a/examples/quick_modeling_test.py b/examples/quick_modeling_test.py new file mode 100644 index 0000000..254611b --- /dev/null +++ b/examples/quick_modeling_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +""" +Quick test script for the modeling API functionality. + +This is a simplified version for quick testing. It makes real API calls +to verify the modeling endpoints are working. + +Set these environment variables: +- CVEC_HOST: Your CVec backend host (e.g., https://your-tenant.cvector.dev) +- CVEC_API_KEY: Your CVec API key + +Usage: + python examples/quick_modeling_test.py +""" + +import os +from datetime import datetime, timedelta + +from cvec import CVec + + +def main(): + """Quick test of modeling API.""" + print("Quick Modeling API Test") + print("=" * 40) + + # Get environment variables + host = os.environ.get("CVEC_HOST") + api_key = os.environ.get("CVEC_API_KEY") + host = "https://sandbox.cvector.dev" + api_key = "cva_1cFKZhgZbRzbVQHQjpGLNMUSqyphh31AiVP8" + + if not host or not api_key: + print("ERROR: Please set CVEC_HOST and CVEC_API_KEY environment variables") + return + + try: + # Initialize client + print("Initializing CVec client...") + cvec = CVec(host=host, api_key=api_key) + print("SUCCESS: Client initialized") + + # Test 1: Basic authentication + print("\nTesting authentication...") + metrics = cvec.get_metrics() + print(f"SUCCESS: Auth OK - got {len(metrics)} metrics") + + # Test 2: Modeling readings + print("\nTesting modeling readings...") + end_date = datetime.now() + start_date = end_date - timedelta(hours=3) # Last hour + + response = cvec.get_modeling_readings( + tag_ids=[63], # Common tag IDs + start_at=start_date, + end_at=end_date, + desired_points=50, + ) + print(response) + + print(f"SUCCESS: Modeling readings OK - got {len(response.items)} tag groups") + for group in response.items: + print(f" Tag {group.tag_id}: {len(group.data)} points") + + # Test 3: Latest readings + # print("\nTesting latest readings...") + # latest = cvec.get_modeling_latest_readings(tag_ids=[1, 2]) + # print(f"SUCCESS: Latest readings OK - got {len(latest.items)} readings") + + print("\nALL TESTS PASSED! Modeling API is working.") + + except Exception as e: + print(f"\nERROR: Test failed: {e}") + print("\nCommon issues:") + print("- Check your API key permissions") + print("- Verify the backend is running") + print("- Check network connectivity") + print("- Use valid tag IDs from your system") + + +if __name__ == "__main__": + main() diff --git a/examples/test_modeling_api_end_to_end.py b/examples/test_modeling_api_end_to_end.py new file mode 100644 index 0000000..9fe8b79 --- /dev/null +++ b/examples/test_modeling_api_end_to_end.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +""" +End-to-end test script for the modeling API functionality. + +This script tests the real API calls to verify that: +1. The modeling endpoints are accessible +2. Authentication works properly +3. Data can be retrieved from the modeling database +4. The response format matches expectations + +Make sure to set the following environment variables: +- CVEC_HOST: Your CVec backend host URL (e.g., https://your-tenant.cvector.dev) +- CVEC_API_KEY: Your CVec API key (must start with 'cva_' and be 40 characters) + +Usage: + python examples/test_modeling_api_end_to_end.py +""" + +import os +import sys +from datetime import datetime, timedelta +from typing import List + +from cvec import CVec +from cvec.models.modeling import ( + ModelingReadingsDataResponse, + ModelingReadingsGroup, + LatestReadingsResponse, +) + + +def test_connection_and_auth(cvec: CVec) -> bool: + """Test basic connection and authentication.""" + print("Testing connection and authentication...") + + try: + # Try to get metrics to verify authentication works + metrics = cvec.get_metrics() + print(f"SUCCESS: Authentication successful! Retrieved {len(metrics)} metrics") + return True + except Exception as e: + print(f"ERROR: Authentication failed: {e}") + return False + + +def test_modeling_readings_endpoint(cvec: CVec) -> bool: + """Test the modeling readings endpoint.""" + print("\nTesting modeling readings endpoint...") + + try: + # Set time range (last 24 hours) + end_date = datetime.now() + start_date = end_date - timedelta(hours=24) + + # Use some common tag IDs (you may need to adjust these) + test_tag_ids = [1, 2, 3] + + print(f" Requesting data for tag IDs: {test_tag_ids}") + print(f" Time range: {start_date} to {end_date}") + + # Make the API call + response = cvec.get_modeling_readings( + tag_ids=test_tag_ids, + start_at=start_date, + end_at=end_date, + desired_points=100, # Small number for testing + ) + + # Verify response structure + assert isinstance(response, ModelingReadingsDataResponse), "Response should be ModelingReadingsDataResponse" + print(f"SUCCESS: Modeling readings endpoint working! Retrieved {len(response.items)} tag groups") + + # Print details about the response + for i, group in enumerate(response.items): + print(f" Tag {group.tag_id}: {len(group.data)} data points, source: {group.source}") + if group.data: + first_point = group.data[0] + last_point = group.data[-1] + print(f" First: {datetime.fromtimestamp(first_point.timestamp)} = {first_point.tag_value}") + print(f" Last: {datetime.fromtimestamp(last_point.timestamp)} = {last_point.tag_value}") + + return True + + except Exception as e: + print(f"ERROR: Modeling readings endpoint failed: {e}") + if "403" in str(e): + print(" This might be a permission issue - check if your API key has VISUALIZATION_TRENDS_READ permission") + elif "404" in str(e): + print(" This might mean the modeling endpoint doesn't exist or isn't accessible") + elif "500" in str(e): + print(" This might be a server-side error - check the backend logs") + return False + + +def test_latest_readings_endpoint(cvec: CVec) -> bool: + """Test the latest modeling readings endpoint.""" + print("\nTesting latest modeling readings endpoint...") + + try: + # Use some common tag IDs (you may need to adjust these) + test_tag_ids = [1, 2, 3] + + print(f" Requesting latest readings for tag IDs: {test_tag_ids}") + + # Make the API call + response = cvec.get_modeling_latest_readings(tag_ids=test_tag_ids) + + # Verify response structure + assert isinstance(response, LatestReadingsResponse), "Response should be LatestReadingsResponse" + print(f"SUCCESS: Latest readings endpoint working! Retrieved {len(response.items)} latest readings") + + # Print details about the response + for item in response.items: + print(f" Tag {item.tag_id}: Value {item.tag_value} at {item.tag_value_changed_at}") + + return True + + except Exception as e: + print(f"ERROR: Latest readings endpoint failed: {e}") + if "403" in str(e): + print(" This might be a permission issue - check if your API key has TAG_DIRECTORIES_READ permission") + return False + + +def test_error_handling(cvec: CVec) -> bool: + """Test error handling with invalid requests.""" + print("\nTesting error handling...") + + try: + # Test with invalid tag IDs (negative numbers) + print(" Testing with invalid tag IDs...") + response = cvec.get_modeling_readings( + tag_ids=[-1, -2], # Invalid negative tag IDs + start_at=datetime.now() - timedelta(hours=1), + end_at=datetime.now(), + desired_points=10, + ) + print(" ERROR: Should have failed with invalid tag IDs") + return False + + except Exception as e: + print(f" SUCCESS: Properly handled invalid request: {e}") + + try: + # Test with invalid date range (end before start) + print(" Testing with invalid date range...") + response = cvec.get_modeling_readings( + tag_ids=[1, 2], + start_at=datetime.now(), + end_at=datetime.now() - timedelta(hours=1), # End before start + desired_points=10, + ) + print(" ERROR: Should have failed with invalid date range") + return False + + except Exception as e: + print(f" SUCCESS: Properly handled invalid date range: {e}") + + return True + + +def main(): + """Main test function.""" + print("Starting end-to-end test of modeling API functionality") + print("=" * 60) + + # Check environment variables + host = os.environ.get("CVEC_HOST") + api_key = os.environ.get("CVEC_API_KEY") + + if not host: + print("ERROR: CVEC_HOST environment variable not set") + print(" Please set it to your CVec backend host (e.g., https://your-tenant.cvector.dev)") + sys.exit(1) + + if not api_key: + print("ERROR: CVEC_API_KEY environment variable not set") + print(" Please set it to your CVec API key (must start with 'cva_' and be 40 characters)") + sys.exit(1) + + if not api_key.startswith("cva_") or len(api_key) != 40: + print("ERROR: Invalid API key format") + print(" API key must start with 'cva_' and be exactly 40 characters") + print(f" Current key: {api_key[:10]}... (length: {len(api_key)})") + sys.exit(1) + + print(f"Host: {host}") + print(f"API Key: {api_key[:10]}...") + print("=" * 60) + + try: + # Initialize CVec client + print("Initializing CVec client...") + cvec = CVec(host=host, api_key=api_key) + print("SUCCESS: CVec client initialized successfully") + + # Run tests + tests = [ + ("Connection & Authentication", test_connection_and_auth), + ("Modeling Readings Endpoint", test_modeling_readings_endpoint), + ("Latest Readings Endpoint", test_latest_readings_endpoint), + ("Error Handling", test_error_handling), + ] + + results = [] + for test_name, test_func in tests: + try: + result = test_func(cvec) + results.append((test_name, result)) + except Exception as e: + print(f"ERROR: Test '{test_name}' crashed: {e}") + results.append((test_name, False)) + + # Print summary + print("\n" + "=" * 60) + print("TEST SUMMARY") + print("=" * 60) + + passed = 0 + total = len(results) + + for test_name, result in results: + status = "PASS" if result else "FAIL" + print(f"{status} {test_name}") + if result: + passed += 1 + + print(f"\nResults: {passed}/{total} tests passed") + + if passed == total: + print("ALL TESTS PASSED! The modeling API is working correctly.") + print("\nYou can now use the modeling functionality in your applications:") + print(" - cvec.get_modeling_readings() for historical data") + print(" - cvec.get_modeling_latest_readings() for current values") + else: + print("Some tests failed. Check the output above for details.") + print(" Common issues:") + print(" - Permission problems (check API key permissions)") + print(" - Network connectivity issues") + print(" - Backend service not running") + print(" - Invalid tag IDs (use actual tag IDs from your system)") + + return passed == total + + except Exception as e: + print(f"ERROR: Fatal error during testing: {e}") + import traceback + traceback.print_exc() + return False + + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1) diff --git a/src/cvec/cvec.py b/src/cvec/cvec.py index 5590515..ff72513 100644 --- a/src/cvec/cvec.py +++ b/src/cvec/cvec.py @@ -11,6 +11,12 @@ arrow_to_metric_data_points, metric_data_points_to_arrow, ) +from cvec.models.modeling import ( + FetchModelingReadingsRequest, + ModelingReadingsDataResponse, + LatestReadingsRequest, + LatestReadingsResponse, +) class CVec: @@ -305,6 +311,59 @@ def add_metric_data( ] self._make_request("POST", endpoint, json=data_dicts) # type: ignore[arg-type] + def get_modeling_readings( + self, + tag_ids: List[int], + start_at: Optional[datetime] = None, + end_at: Optional[datetime] = None, + desired_points: int = 10000, + ) -> ModelingReadingsDataResponse: + """ + Fetch modeling readings for chosen tags. + Perform aggregation to desired count and downsampling via TimescaleDB mechanisms. + + Args: + tag_ids: Tag ID list to fetch data for + start_at: Optional start time for the query (uses class default if not specified) + end_at: Optional end time for the query (uses class default if not specified) + desired_points: Number of data points to return (default: 10000) + + Returns: + ModelingReadingsDataResponse containing readings for each tag + """ + _start_at = start_at or self.default_start_at + _end_at = end_at or self.default_end_at + + request_data = FetchModelingReadingsRequest( + tag_ids=tag_ids, + start_date=_start_at, + end_date=_end_at, + desired_points=desired_points, + ) + + response_data = self._make_request( + "POST", "/api/modeling/fetch_modeling_data", json=request_data.model_dump(mode="json") + ) + print(response_data) + return ModelingReadingsDataResponse.model_validate(response_data) + + def get_modeling_latest_readings(self, tag_ids: List[int]) -> LatestReadingsResponse: + """ + Fetch single latest reading for each of specified tag IDs from the modeling database. + + Args: + tag_ids: List of tag IDs to fetch data for + + Returns: + LatestReadingsResponse containing latest readings + """ + request_data = LatestReadingsRequest(tag_ids=tag_ids) + + response_data = self._make_request( + "POST", "/api/modeling/fetch_latest_readings", json=request_data.model_dump(mode="json") + ) + return LatestReadingsResponse.model_validate(response_data) + def _login_with_supabase(self, email: str, password: str) -> None: """ Login to Supabase and get access/refresh tokens. diff --git a/src/cvec/models/__init__.py b/src/cvec/models/__init__.py index d986a8d..703120c 100644 --- a/src/cvec/models/__init__.py +++ b/src/cvec/models/__init__.py @@ -1,4 +1,26 @@ from .metric import Metric, MetricDataPoint +from .modeling import ( + FetchModelingReadingsRequest, + LatestReadingsRequest, + LatestReadingsResponse, + LatestReadingsResponseItem, + ModelingReadingsDataResponse, + ModelingReadingsGroup, + ModelingReadingModel, + TagSourceType, +) from .span import Span -__all__ = ["Metric", "MetricDataPoint", "Span"] +__all__ = [ + "Metric", + "MetricDataPoint", + "Span", + "ModelingReadingsDataResponse", + "ModelingReadingsGroup", + "ModelingReadingModel", + "FetchModelingReadingsRequest", + "LatestReadingsRequest", + "LatestReadingsResponse", + "LatestReadingsResponseItem", + "TagSourceType", +] diff --git a/src/cvec/models/modeling.py b/src/cvec/models/modeling.py new file mode 100644 index 0000000..dd27568 --- /dev/null +++ b/src/cvec/models/modeling.py @@ -0,0 +1,55 @@ +from datetime import datetime +from enum import Enum +from typing import List + +from pydantic import BaseModel + + +class TagSourceType(str, Enum): + Sensor = "sensor_readings" + Modeling = "modeling" + + +class BaseReadingModel(BaseModel): + tag_id: int + tag_value: float + timestamp: float + + +class ModelingReadingModel(BaseReadingModel): + pass + + +class BaseReadingsGroup(BaseModel): + tag_id: int + data: List[BaseReadingModel] + source: TagSourceType + + +class ModelingReadingsGroup(BaseReadingsGroup): + pass + + +class ModelingReadingsDataResponse(BaseModel): + items: List[ModelingReadingsGroup] + + +class FetchModelingReadingsRequest(BaseModel): + tag_ids: List[int] + start_date: datetime + end_date: datetime + desired_points: int = 10000 + + +class LatestReadingsRequest(BaseModel): + tag_ids: List[int] + + +class LatestReadingsResponseItem(BaseModel): + tag_id: int + tag_value: float + tag_value_changed_at: datetime + + +class LatestReadingsResponse(BaseModel): + items: List[LatestReadingsResponseItem] diff --git a/tests/test_modeling.py b/tests/test_modeling.py new file mode 100644 index 0000000..4054e05 --- /dev/null +++ b/tests/test_modeling.py @@ -0,0 +1,209 @@ +""" +Tests for the modeling functionality in the CVec client. +""" + +import pytest +from datetime import datetime +from unittest.mock import Mock, patch + +from cvec.models.modeling import ( + FetchModelingReadingsRequest, + ModelingReadingsDataResponse, + ModelingReadingsGroup, + ModelingReadingModel, + LatestReadingsRequest, + LatestReadingsResponse, + LatestReadingsResponseItem, + TagSourceType, +) +from cvec.cvec import CVec + + +class TestModelingModels: + """Test the modeling data models.""" + + def test_fetch_modeling_readings_request(self): + """Test FetchModelingReadingsRequest model.""" + request = FetchModelingReadingsRequest( + tag_ids=[1, 2, 3], + start_date=datetime(2024, 1, 1, 12, 0, 0), + end_date=datetime(2024, 1, 1, 13, 0, 0), + desired_points=1000, + ) + + assert request.tag_ids == [1, 2, 3] + assert request.desired_points == 1000 + assert request.model_dump()["tag_ids"] == [1, 2, 3] + + def test_modeling_reading_model(self): + """Test ModelingReadingModel model.""" + reading = ModelingReadingModel( + tag_id=1, + tag_value=42.5, + timestamp=1704110400.0, # 2024-01-01 12:00:00 UTC + ) + + assert reading.tag_id == 1 + assert reading.tag_value == 42.5 + assert reading.timestamp == 1704110400.0 + + def test_modeling_readings_group(self): + """Test ModelingReadingsGroup model.""" + readings = [ + ModelingReadingModel(tag_id=1, tag_value=10.0, timestamp=1704110400.0), + ModelingReadingModel(tag_id=1, tag_value=15.0, timestamp=1704114000.0), + ] + + group = ModelingReadingsGroup( + tag_id=1, + data=readings, + source=TagSourceType.Modeling, + ) + + assert group.tag_id == 1 + assert len(group.data) == 2 + assert group.source == TagSourceType.Modeling + + def test_modeling_readings_data_response(self): + """Test ModelingReadingsDataResponse model.""" + readings = [ + ModelingReadingsGroup( + tag_id=1, + data=[ + ModelingReadingModel(tag_id=1, tag_value=10.0, timestamp=1704110400.0), + ], + source=TagSourceType.Modeling, + ), + ] + + response = ModelingReadingsDataResponse(items=readings) + assert len(response.items) == 1 + assert response.items[0].tag_id == 1 + + def test_latest_readings_request(self): + """Test LatestReadingsRequest model.""" + request = LatestReadingsRequest(tag_ids=[1, 2, 3]) + assert request.tag_ids == [1, 2, 3] + + def test_latest_readings_response(self): + """Test LatestReadingsResponse model.""" + items = [ + LatestReadingsResponseItem( + tag_id=1, + tag_value=42.5, + tag_value_changed_at=datetime(2024, 1, 1, 12, 0, 0), + ), + ] + + response = LatestReadingsResponse(items=items) + assert len(response.items) == 1 + assert response.items[0].tag_id == 1 + assert response.items[0].tag_value == 42.5 + + +class TestModelingMethods: + """Test the modeling methods in the CVec class.""" + + @patch('cvec.cvec.CVec._fetch_publishable_key') + @patch('cvec.cvec.CVec._login_with_supabase') + @patch('cvec.cvec.CVec._make_request') + def test_get_modeling_readings(self, mock_make_request, mock_login, mock_fetch_key): + """Test get_modeling_readings method.""" + # Mock the publishable key fetch + mock_fetch_key.return_value = "test_publishable_key" + + # Mock the login method + mock_login.return_value = None + + # Mock the response + mock_response = { + "items": [ + { + "tag_id": 1, + "data": [ + { + "tag_id": 1, + "tag_value": 10.0, + "timestamp": 1704110400.0, + } + ], + "source": "modeling", + } + ] + } + mock_make_request.return_value = mock_response + + # Create CVec instance + cvec = CVec(host="http://test.com", api_key="cva_test12345678901234567890123456789012") + + # Call the method + start_date = datetime(2024, 1, 1, 12, 0, 0) + end_date = datetime(2024, 1, 1, 13, 0, 0) + result = cvec.get_modeling_readings( + tag_ids=[1], + start_at=start_date, + end_at=end_date, + desired_points=1000, + ) + + # Verify the result + assert isinstance(result, ModelingReadingsDataResponse) + assert len(result.items) == 1 + assert result.items[0].tag_id == 1 + assert result.items[0].source == TagSourceType.Modeling + + # Verify the request was made correctly + mock_make_request.assert_called_once() + call_args = mock_make_request.call_args + assert call_args[0][0] == "POST" + assert call_args[0][1] == "/api/modeling/fetch_modeling_data" + assert call_args[1]["json"]["tag_ids"] == [1] + assert call_args[1]["json"]["start_date"] == "2024-01-01T12:00:00" + assert call_args[1]["json"]["end_date"] == "2024-01-01T13:00:00" + assert call_args[1]["json"]["desired_points"] == 1000 + + @patch('cvec.cvec.CVec._fetch_publishable_key') + @patch('cvec.cvec.CVec._login_with_supabase') + @patch('cvec.cvec.CVec._make_request') + def test_get_modeling_latest_readings(self, mock_make_request, mock_login, mock_fetch_key): + """Test get_modeling_latest_readings method.""" + # Mock the publishable key fetch + mock_fetch_key.return_value = "test_publishable_key" + + # Mock the login method + mock_login.return_value = None + + # Mock the response + mock_response = { + "items": [ + { + "tag_id": 1, + "tag_value": 42.5, + "tag_value_changed_at": "2024-01-01T12:00:00", + } + ] + } + mock_make_request.return_value = mock_response + + # Create CVec instance + cvec = CVec(host="http://test.com", api_key="cva_test12345678901234567890123456789012") + + # Call the method + result = cvec.get_modeling_latest_readings(tag_ids=[1]) + + # Verify the result + assert isinstance(result, LatestReadingsResponse) + assert len(result.items) == 1 + assert result.items[0].tag_id == 1 + assert result.items[0].tag_value == 42.5 + + # Verify the request was made correctly + mock_make_request.assert_called_once() + call_args = mock_make_request.call_args + assert call_args[0][0] == "POST" + assert call_args[0][1] == "/api/modeling/fetch_latest_readings" + assert call_args[1]["json"]["tag_ids"] == [1] + + +if __name__ == "__main__": + pytest.main([__file__]) From c68510300f3d0d517c8ec47f936c97bd189af3d0 Mon Sep 17 00:00:00 2001 From: Amy Ni Date: Tue, 2 Sep 2025 18:33:41 -0400 Subject: [PATCH 2/4] feat: added methods to fetch the modeling data from our backend --- README.md | 29 ++ examples/get_modeling_data_example.py | 102 ------- ...get_modeling_metrics_data_arrow_example.py | 26 ++ examples/get_modeling_metrics_data_example.py | 33 +++ examples/get_modeling_metrics_example.py | 18 ++ examples/quick_modeling_test.py | 82 ------ examples/test_modeling_api_end_to_end.py | 254 ---------------- src/cvec/cvec.py | 98 ++++--- tests/test_modeling.py | 277 ++++++++---------- 9 files changed, 288 insertions(+), 631 deletions(-) delete mode 100644 examples/get_modeling_data_example.py create mode 100644 examples/get_modeling_metrics_data_arrow_example.py create mode 100644 examples/get_modeling_metrics_data_example.py create mode 100644 examples/get_modeling_metrics_example.py delete mode 100644 examples/quick_modeling_test.py delete mode 100644 examples/test_modeling_api_end_to_end.py diff --git a/README.md b/README.md index 0d4890c..3111e68 100644 --- a/README.md +++ b/README.md @@ -232,3 +232,32 @@ Add multiple metric data points to the database. ## `get_metrics(?start_at, ?end_at)` Return a list of metrics that had at least one transition in the given [`start_at`, `end_at`) interval. All metrics are returned if no `start_at` and `end_at` are given. + +## `get_modeling_metrics(start_at, end_at)` + +Fetch modeling metrics from the modeling database. This method returns a list of available modeling metrics that had transitions in the specified time range. + +- `start_at`: Optional start date for the query range (uses class default if not specified) +- `end_at`: Optional end date for the query range (uses class default if not specified) + +Returns a list of `Metric` objects containing modeling metrics. + +## `get_modeling_metrics_data(names, start_at, end_at)` + +Fetch actual data values from modeling metrics within a time range. This method returns the actual data points (values) for the specified modeling metrics, similar to `get_metric_data()` but for the modeling database. + +- `names`: Optional list of modeling metric names to filter by +- `start_at`: Optional start time for the query (uses class default if not specified) +- `end_at`: Optional end time for the query (uses class default if not specified) + +Returns a list of `MetricDataPoint` objects containing the actual data values. + +## `get_modeling_metrics_data_arrow(names, start_at, end_at)` + +Fetch actual data values from modeling metrics within a time range in Apache Arrow format. This method returns the actual data points (values) for the specified modeling metrics in Arrow IPC format, which is more efficient for large datasets. + +- `names`: Optional list of modeling metric names to filter by +- `start_at`: Optional start time for the query (uses class default if not specified) +- `end_at`: Optional end time for the query (uses class default if not specified) + +Returns Arrow IPC format data that can be read using `pyarrow.ipc.open_file()`. diff --git a/examples/get_modeling_data_example.py b/examples/get_modeling_data_example.py deleted file mode 100644 index 56642f3..0000000 --- a/examples/get_modeling_data_example.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -""" -Example script demonstrating how to retrieve modeling data from the CVec API. - -This script shows how to: -1. Fetch modeling readings for specific tag IDs within a time range -2. Get the latest modeling readings for specific tag IDs -3. Handle the response data - -Make sure to set the following environment variables: -- CVEC_HOST: Your CVec backend host URL -- CVEC_API_KEY: Your CVec API key -""" - -import os -from datetime import datetime, timedelta -from typing import List - -from cvec import CVec -from cvec.models.modeling import ModelingReadingsGroup, ModelingReadingModel - - -def print_modeling_readings(readings: List[ModelingReadingsGroup]) -> None: - """Print modeling readings in a formatted way.""" - print(f"Retrieved {len(readings)} tag groups:") - print("-" * 50) - - for group in readings: - print(f"Tag ID: {group.tag_id}") - print(f"Source: {group.source}") - print(f"Number of data points: {len(group.data)}") - - if group.data: - # Show first and last few data points - print(" First few data points:") - for i, point in enumerate(group.data[:3]): - timestamp = datetime.fromtimestamp(point.timestamp) - print(f" {i+1}. Time: {timestamp}, Value: {point.tag_value}") - - if len(group.data) > 3: - print(f" ... and {len(group.data) - 3} more points") - - # Show last data point - last_point = group.data[-1] - last_timestamp = datetime.fromtimestamp(last_point.timestamp) - print(f" Last data point: Time: {last_timestamp}, Value: {last_point.tag_value}") - - print("-" * 30) - - -def main(): - """Main function demonstrating modeling data retrieval.""" - try: - # Initialize CVec client - cvec = CVec() - print(f"Connected to CVec at: {cvec.host}") - - # Example tag IDs (replace with actual tag IDs from your system) - tag_ids = [1, 2, 3] - - # Set time range (last 24 hours) - end_date = datetime.now() - start_date = end_date - timedelta(hours=24) - - print(f"Fetching modeling data for tag IDs: {tag_ids}") - print(f"Time range: {start_date} to {end_date}") - print("=" * 60) - - # Fetch modeling readings - print("1. Fetching modeling readings...") - modeling_response = cvec.get_modeling_readings( - tag_ids=tag_ids, - start_at=start_date, - end_at=end_date, - desired_points=1000, # Limit to 1000 points for performance - ) - - print(f"Successfully retrieved modeling data with {len(modeling_response.items)} tag groups") - print_modeling_readings(modeling_response.items) - - print("\n" + "=" * 60) - - # Get latest modeling readings - print("2. Fetching latest modeling readings...") - latest_response = cvec.get_modeling_latest_readings(tag_ids=tag_ids) - - print(f"Successfully retrieved latest readings for {len(latest_response.items)} tags:") - for item in latest_response.items: - print(f" Tag {item.tag_id}: Value {item.tag_value} at {item.tag_value_changed_at}") - - print("\n" + "=" * 60) - print("Example completed successfully!") - - except Exception as e: - print(f"Error occurred: {e}") - print("Make sure your environment variables are set correctly:") - print("- CVEC_HOST: Your CVec backend host URL") - print("- CVEC_API_KEY: Your CVec API key") - - -if __name__ == "__main__": - main() diff --git a/examples/get_modeling_metrics_data_arrow_example.py b/examples/get_modeling_metrics_data_arrow_example.py new file mode 100644 index 0000000..6af3c43 --- /dev/null +++ b/examples/get_modeling_metrics_data_arrow_example.py @@ -0,0 +1,26 @@ +from cvec import CVec +import io +import pyarrow.ipc as ipc # type: ignore[import-untyped] +import os + + +def main() -> None: + cvec = CVec( + host=os.environ.get("CVEC_HOST", "https://your-subdomain.cvector.dev"), + api_key=os.environ.get("CVEC_API_KEY", "your-api-key"), + ) + test_metric_name = "Data_Marketplace/PROD/Miso_5min/ILLINOIS_HUB_lmp" + print("\nGetting modeling metrics data as Arrow...") + arrow_data = cvec.get_modeling_metrics_data_arrow(names=[test_metric_name]) + reader = ipc.open_file(io.BytesIO(arrow_data)) + table = reader.read_all() + print(f"Arrow table shape: {len(table)} rows") + print("\nFirst few rows:") + for i in range(min(5, len(table))): + print( + f"- {table['name'][i].as_py()}: {table['value_double'][i].as_py() or table['value_string'][i].as_py()} at {table['time'][i].as_py()}" + ) + + +if __name__ == "__main__": + main() diff --git a/examples/get_modeling_metrics_data_example.py b/examples/get_modeling_metrics_data_example.py new file mode 100644 index 0000000..36ba526 --- /dev/null +++ b/examples/get_modeling_metrics_data_example.py @@ -0,0 +1,33 @@ +from cvec import CVec +import os +from datetime import datetime, timedelta + + +def main() -> None: + cvec = CVec( + host=os.environ.get("CVEC_HOST", "https://your-subdomain.cvector.dev"), + api_key=os.environ.get("CVEC_API_KEY", "your-api-key"), + ) + end_date = datetime.now() + start_date = end_date - timedelta(hours=3) + print("\nGetting modeling metrics data...") + modeling_data = cvec.get_modeling_metrics_data( + names=["Data_Marketplace/PROD/Miso_5min/ILLINOIS_HUB_lmp"], + start_at=start_date, + end_at=end_date, + ) + print(f"Found {len(modeling_data)} data points") + + if modeling_data: + print("\nFirst few data points:") + for i, point in enumerate(modeling_data[:5]): + print( + f"- {point.name}: {point.value_double or point.value_string} at {point.time}" + ) + + if len(modeling_data) > 5: + print(f"... and {len(modeling_data) - 5} more data points") + + +if __name__ == "__main__": + main() diff --git a/examples/get_modeling_metrics_example.py b/examples/get_modeling_metrics_example.py new file mode 100644 index 0000000..711e7fe --- /dev/null +++ b/examples/get_modeling_metrics_example.py @@ -0,0 +1,18 @@ +from cvec import CVec +import os + + +def main() -> None: + cvec = CVec( + host=os.environ.get("CVEC_HOST", "https://your-subdomain.cvector.dev"), + api_key=os.environ.get("CVEC_API_KEY", "your-api-key"), + ) + print("\nGetting available modeling metrics...") + modeling_metrics = cvec.get_modeling_metrics() + print(f"Found {len(modeling_metrics)} modeling metrics") + for metric in modeling_metrics: + print(f"- {metric.name}") + + +if __name__ == "__main__": + main() diff --git a/examples/quick_modeling_test.py b/examples/quick_modeling_test.py deleted file mode 100644 index 254611b..0000000 --- a/examples/quick_modeling_test.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -""" -Quick test script for the modeling API functionality. - -This is a simplified version for quick testing. It makes real API calls -to verify the modeling endpoints are working. - -Set these environment variables: -- CVEC_HOST: Your CVec backend host (e.g., https://your-tenant.cvector.dev) -- CVEC_API_KEY: Your CVec API key - -Usage: - python examples/quick_modeling_test.py -""" - -import os -from datetime import datetime, timedelta - -from cvec import CVec - - -def main(): - """Quick test of modeling API.""" - print("Quick Modeling API Test") - print("=" * 40) - - # Get environment variables - host = os.environ.get("CVEC_HOST") - api_key = os.environ.get("CVEC_API_KEY") - host = "https://sandbox.cvector.dev" - api_key = "cva_1cFKZhgZbRzbVQHQjpGLNMUSqyphh31AiVP8" - - if not host or not api_key: - print("ERROR: Please set CVEC_HOST and CVEC_API_KEY environment variables") - return - - try: - # Initialize client - print("Initializing CVec client...") - cvec = CVec(host=host, api_key=api_key) - print("SUCCESS: Client initialized") - - # Test 1: Basic authentication - print("\nTesting authentication...") - metrics = cvec.get_metrics() - print(f"SUCCESS: Auth OK - got {len(metrics)} metrics") - - # Test 2: Modeling readings - print("\nTesting modeling readings...") - end_date = datetime.now() - start_date = end_date - timedelta(hours=3) # Last hour - - response = cvec.get_modeling_readings( - tag_ids=[63], # Common tag IDs - start_at=start_date, - end_at=end_date, - desired_points=50, - ) - print(response) - - print(f"SUCCESS: Modeling readings OK - got {len(response.items)} tag groups") - for group in response.items: - print(f" Tag {group.tag_id}: {len(group.data)} points") - - # Test 3: Latest readings - # print("\nTesting latest readings...") - # latest = cvec.get_modeling_latest_readings(tag_ids=[1, 2]) - # print(f"SUCCESS: Latest readings OK - got {len(latest.items)} readings") - - print("\nALL TESTS PASSED! Modeling API is working.") - - except Exception as e: - print(f"\nERROR: Test failed: {e}") - print("\nCommon issues:") - print("- Check your API key permissions") - print("- Verify the backend is running") - print("- Check network connectivity") - print("- Use valid tag IDs from your system") - - -if __name__ == "__main__": - main() diff --git a/examples/test_modeling_api_end_to_end.py b/examples/test_modeling_api_end_to_end.py deleted file mode 100644 index 9fe8b79..0000000 --- a/examples/test_modeling_api_end_to_end.py +++ /dev/null @@ -1,254 +0,0 @@ -#!/usr/bin/env python3 -""" -End-to-end test script for the modeling API functionality. - -This script tests the real API calls to verify that: -1. The modeling endpoints are accessible -2. Authentication works properly -3. Data can be retrieved from the modeling database -4. The response format matches expectations - -Make sure to set the following environment variables: -- CVEC_HOST: Your CVec backend host URL (e.g., https://your-tenant.cvector.dev) -- CVEC_API_KEY: Your CVec API key (must start with 'cva_' and be 40 characters) - -Usage: - python examples/test_modeling_api_end_to_end.py -""" - -import os -import sys -from datetime import datetime, timedelta -from typing import List - -from cvec import CVec -from cvec.models.modeling import ( - ModelingReadingsDataResponse, - ModelingReadingsGroup, - LatestReadingsResponse, -) - - -def test_connection_and_auth(cvec: CVec) -> bool: - """Test basic connection and authentication.""" - print("Testing connection and authentication...") - - try: - # Try to get metrics to verify authentication works - metrics = cvec.get_metrics() - print(f"SUCCESS: Authentication successful! Retrieved {len(metrics)} metrics") - return True - except Exception as e: - print(f"ERROR: Authentication failed: {e}") - return False - - -def test_modeling_readings_endpoint(cvec: CVec) -> bool: - """Test the modeling readings endpoint.""" - print("\nTesting modeling readings endpoint...") - - try: - # Set time range (last 24 hours) - end_date = datetime.now() - start_date = end_date - timedelta(hours=24) - - # Use some common tag IDs (you may need to adjust these) - test_tag_ids = [1, 2, 3] - - print(f" Requesting data for tag IDs: {test_tag_ids}") - print(f" Time range: {start_date} to {end_date}") - - # Make the API call - response = cvec.get_modeling_readings( - tag_ids=test_tag_ids, - start_at=start_date, - end_at=end_date, - desired_points=100, # Small number for testing - ) - - # Verify response structure - assert isinstance(response, ModelingReadingsDataResponse), "Response should be ModelingReadingsDataResponse" - print(f"SUCCESS: Modeling readings endpoint working! Retrieved {len(response.items)} tag groups") - - # Print details about the response - for i, group in enumerate(response.items): - print(f" Tag {group.tag_id}: {len(group.data)} data points, source: {group.source}") - if group.data: - first_point = group.data[0] - last_point = group.data[-1] - print(f" First: {datetime.fromtimestamp(first_point.timestamp)} = {first_point.tag_value}") - print(f" Last: {datetime.fromtimestamp(last_point.timestamp)} = {last_point.tag_value}") - - return True - - except Exception as e: - print(f"ERROR: Modeling readings endpoint failed: {e}") - if "403" in str(e): - print(" This might be a permission issue - check if your API key has VISUALIZATION_TRENDS_READ permission") - elif "404" in str(e): - print(" This might mean the modeling endpoint doesn't exist or isn't accessible") - elif "500" in str(e): - print(" This might be a server-side error - check the backend logs") - return False - - -def test_latest_readings_endpoint(cvec: CVec) -> bool: - """Test the latest modeling readings endpoint.""" - print("\nTesting latest modeling readings endpoint...") - - try: - # Use some common tag IDs (you may need to adjust these) - test_tag_ids = [1, 2, 3] - - print(f" Requesting latest readings for tag IDs: {test_tag_ids}") - - # Make the API call - response = cvec.get_modeling_latest_readings(tag_ids=test_tag_ids) - - # Verify response structure - assert isinstance(response, LatestReadingsResponse), "Response should be LatestReadingsResponse" - print(f"SUCCESS: Latest readings endpoint working! Retrieved {len(response.items)} latest readings") - - # Print details about the response - for item in response.items: - print(f" Tag {item.tag_id}: Value {item.tag_value} at {item.tag_value_changed_at}") - - return True - - except Exception as e: - print(f"ERROR: Latest readings endpoint failed: {e}") - if "403" in str(e): - print(" This might be a permission issue - check if your API key has TAG_DIRECTORIES_READ permission") - return False - - -def test_error_handling(cvec: CVec) -> bool: - """Test error handling with invalid requests.""" - print("\nTesting error handling...") - - try: - # Test with invalid tag IDs (negative numbers) - print(" Testing with invalid tag IDs...") - response = cvec.get_modeling_readings( - tag_ids=[-1, -2], # Invalid negative tag IDs - start_at=datetime.now() - timedelta(hours=1), - end_at=datetime.now(), - desired_points=10, - ) - print(" ERROR: Should have failed with invalid tag IDs") - return False - - except Exception as e: - print(f" SUCCESS: Properly handled invalid request: {e}") - - try: - # Test with invalid date range (end before start) - print(" Testing with invalid date range...") - response = cvec.get_modeling_readings( - tag_ids=[1, 2], - start_at=datetime.now(), - end_at=datetime.now() - timedelta(hours=1), # End before start - desired_points=10, - ) - print(" ERROR: Should have failed with invalid date range") - return False - - except Exception as e: - print(f" SUCCESS: Properly handled invalid date range: {e}") - - return True - - -def main(): - """Main test function.""" - print("Starting end-to-end test of modeling API functionality") - print("=" * 60) - - # Check environment variables - host = os.environ.get("CVEC_HOST") - api_key = os.environ.get("CVEC_API_KEY") - - if not host: - print("ERROR: CVEC_HOST environment variable not set") - print(" Please set it to your CVec backend host (e.g., https://your-tenant.cvector.dev)") - sys.exit(1) - - if not api_key: - print("ERROR: CVEC_API_KEY environment variable not set") - print(" Please set it to your CVec API key (must start with 'cva_' and be 40 characters)") - sys.exit(1) - - if not api_key.startswith("cva_") or len(api_key) != 40: - print("ERROR: Invalid API key format") - print(" API key must start with 'cva_' and be exactly 40 characters") - print(f" Current key: {api_key[:10]}... (length: {len(api_key)})") - sys.exit(1) - - print(f"Host: {host}") - print(f"API Key: {api_key[:10]}...") - print("=" * 60) - - try: - # Initialize CVec client - print("Initializing CVec client...") - cvec = CVec(host=host, api_key=api_key) - print("SUCCESS: CVec client initialized successfully") - - # Run tests - tests = [ - ("Connection & Authentication", test_connection_and_auth), - ("Modeling Readings Endpoint", test_modeling_readings_endpoint), - ("Latest Readings Endpoint", test_latest_readings_endpoint), - ("Error Handling", test_error_handling), - ] - - results = [] - for test_name, test_func in tests: - try: - result = test_func(cvec) - results.append((test_name, result)) - except Exception as e: - print(f"ERROR: Test '{test_name}' crashed: {e}") - results.append((test_name, False)) - - # Print summary - print("\n" + "=" * 60) - print("TEST SUMMARY") - print("=" * 60) - - passed = 0 - total = len(results) - - for test_name, result in results: - status = "PASS" if result else "FAIL" - print(f"{status} {test_name}") - if result: - passed += 1 - - print(f"\nResults: {passed}/{total} tests passed") - - if passed == total: - print("ALL TESTS PASSED! The modeling API is working correctly.") - print("\nYou can now use the modeling functionality in your applications:") - print(" - cvec.get_modeling_readings() for historical data") - print(" - cvec.get_modeling_latest_readings() for current values") - else: - print("Some tests failed. Check the output above for details.") - print(" Common issues:") - print(" - Permission problems (check API key permissions)") - print(" - Network connectivity issues") - print(" - Backend service not running") - print(" - Invalid tag IDs (use actual tag IDs from your system)") - - return passed == total - - except Exception as e: - print(f"ERROR: Fatal error during testing: {e}") - import traceback - traceback.print_exc() - return False - - -if __name__ == "__main__": - success = main() - sys.exit(0 if success else 1) diff --git a/src/cvec/cvec.py b/src/cvec/cvec.py index ff72513..ec37f92 100644 --- a/src/cvec/cvec.py +++ b/src/cvec/cvec.py @@ -11,12 +11,6 @@ arrow_to_metric_data_points, metric_data_points_to_arrow, ) -from cvec.models.modeling import ( - FetchModelingReadingsRequest, - ModelingReadingsDataResponse, - LatestReadingsRequest, - LatestReadingsResponse, -) class CVec: @@ -311,58 +305,96 @@ def add_metric_data( ] self._make_request("POST", endpoint, json=data_dicts) # type: ignore[arg-type] - def get_modeling_readings( + def get_modeling_metrics( self, - tag_ids: List[int], start_at: Optional[datetime] = None, end_at: Optional[datetime] = None, - desired_points: int = 10000, - ) -> ModelingReadingsDataResponse: + ) -> List[Metric]: """ - Fetch modeling readings for chosen tags. - Perform aggregation to desired count and downsampling via TimescaleDB mechanisms. + Return a list of modeling metrics that had at least one transition in the given [start_at, end_at) interval. + All metrics are returned if no start_at and end_at are given. Args: - tag_ids: Tag ID list to fetch data for start_at: Optional start time for the query (uses class default if not specified) end_at: Optional end time for the query (uses class default if not specified) - desired_points: Number of data points to return (default: 10000) Returns: - ModelingReadingsDataResponse containing readings for each tag + List of Metric objects containing modeling metrics """ _start_at = start_at or self.default_start_at _end_at = end_at or self.default_end_at - request_data = FetchModelingReadingsRequest( - tag_ids=tag_ids, - start_date=_start_at, - end_date=_end_at, - desired_points=desired_points, - ) + params: Dict[str, Any] = { + "start_at": _start_at.isoformat() if _start_at else None, + "end_at": _end_at.isoformat() if _end_at else None, + } response_data = self._make_request( - "POST", "/api/modeling/fetch_modeling_data", json=request_data.model_dump(mode="json") + "GET", "/api/modeling/metrics", params=params ) - print(response_data) - return ModelingReadingsDataResponse.model_validate(response_data) + return [Metric.model_validate(metric_data) for metric_data in response_data] - def get_modeling_latest_readings(self, tag_ids: List[int]) -> LatestReadingsResponse: + def get_modeling_metrics_data( + self, + names: Optional[List[str]] = None, + start_at: Optional[datetime] = None, + end_at: Optional[datetime] = None, + ) -> List[MetricDataPoint]: """ - Fetch single latest reading for each of specified tag IDs from the modeling database. + Return all data-points within a given [start_at, end_at) interval, + optionally selecting a given list of modeling metric names. + Returns a list of MetricDataPoint objects, one for each metric value transition. Args: - tag_ids: List of tag IDs to fetch data for - - Returns: - LatestReadingsResponse containing latest readings + names: Optional list of modeling metric names to filter by + start_at: Optional start time for the query + end_at: Optional end time for the query """ - request_data = LatestReadingsRequest(tag_ids=tag_ids) + _start_at = start_at or self.default_start_at + _end_at = end_at or self.default_end_at + + params: Dict[str, Any] = { + "start_at": _start_at.isoformat() if _start_at else None, + "end_at": _end_at.isoformat() if _end_at else None, + "names": ",".join(names) if names else None, + } response_data = self._make_request( - "POST", "/api/modeling/fetch_latest_readings", json=request_data.model_dump(mode="json") + "GET", "/api/modeling/metrics/data", params=params ) - return LatestReadingsResponse.model_validate(response_data) + return [ + MetricDataPoint.model_validate(point_data) for point_data in response_data + ] + + def get_modeling_metrics_data_arrow( + self, + names: Optional[List[str]] = None, + start_at: Optional[datetime] = None, + end_at: Optional[datetime] = None, + ) -> bytes: + """ + Return all data-points within a given [start_at, end_at) interval, + optionally selecting a given list of modeling metric names. + Returns Arrow IPC format data that can be read using pyarrow.ipc.open_file. + + Args: + names: Optional list of modeling metric names to filter by + start_at: Optional start time for the query + end_at: Optional end time for the query + """ + _start_at = start_at or self.default_start_at + _end_at = end_at or self.default_end_at + + params: Dict[str, Any] = { + "start_at": _start_at.isoformat() if _start_at else None, + "end_at": _end_at.isoformat() if _end_at else None, + "names": ",".join(names) if names else None, + } + + endpoint = "/api/modeling/metrics/data/arrow" + result = self._make_request("GET", endpoint, params=params) + assert isinstance(result, bytes) + return result def _login_with_supabase(self, email: str, password: str) -> None: """ diff --git a/tests/test_modeling.py b/tests/test_modeling.py index 4054e05..8eea9bd 100644 --- a/tests/test_modeling.py +++ b/tests/test_modeling.py @@ -6,203 +6,160 @@ from datetime import datetime from unittest.mock import Mock, patch -from cvec.models.modeling import ( - FetchModelingReadingsRequest, - ModelingReadingsDataResponse, - ModelingReadingsGroup, - ModelingReadingModel, - LatestReadingsRequest, - LatestReadingsResponse, - LatestReadingsResponseItem, - TagSourceType, -) + from cvec.cvec import CVec -class TestModelingModels: - """Test the modeling data models.""" +class TestModelingMethods: + """Test the modeling methods in the CVec class.""" - def test_fetch_modeling_readings_request(self): - """Test FetchModelingReadingsRequest model.""" - request = FetchModelingReadingsRequest( - tag_ids=[1, 2, 3], - start_date=datetime(2024, 1, 1, 12, 0, 0), - end_date=datetime(2024, 1, 1, 13, 0, 0), - desired_points=1000, - ) - - assert request.tag_ids == [1, 2, 3] - assert request.desired_points == 1000 - assert request.model_dump()["tag_ids"] == [1, 2, 3] - - def test_modeling_reading_model(self): - """Test ModelingReadingModel model.""" - reading = ModelingReadingModel( - tag_id=1, - tag_value=42.5, - timestamp=1704110400.0, # 2024-01-01 12:00:00 UTC - ) - - assert reading.tag_id == 1 - assert reading.tag_value == 42.5 - assert reading.timestamp == 1704110400.0 - - def test_modeling_readings_group(self): - """Test ModelingReadingsGroup model.""" - readings = [ - ModelingReadingModel(tag_id=1, tag_value=10.0, timestamp=1704110400.0), - ModelingReadingModel(tag_id=1, tag_value=15.0, timestamp=1704114000.0), + @patch("cvec.cvec.CVec._fetch_publishable_key") + @patch("cvec.cvec.CVec._login_with_supabase") + @patch("cvec.cvec.CVec._make_request") + def test_get_modeling_metrics( + self, mock_make_request: Mock, mock_login: Mock, mock_fetch_key: Mock + ) -> None: + """Test get_modeling_metrics method.""" + # Mock the publishable key fetch + mock_fetch_key.return_value = "test_publishable_key" + + # Mock the login method + mock_login.return_value = None + + # Mock the response + mock_response = [ + { + "id": 1, + "name": "test_metric", + "birth_at": "2024-01-01T12:00:00", + "death_at": None, + } ] - - group = ModelingReadingsGroup( - tag_id=1, - data=readings, - source=TagSourceType.Modeling, + mock_make_request.return_value = mock_response + + # Create CVec instance + cvec = CVec( + host="http://test.com", api_key="cva_test12345678901234567890123456789012" ) - - assert group.tag_id == 1 - assert len(group.data) == 2 - assert group.source == TagSourceType.Modeling - - def test_modeling_readings_data_response(self): - """Test ModelingReadingsDataResponse model.""" - readings = [ - ModelingReadingsGroup( - tag_id=1, - data=[ - ModelingReadingModel(tag_id=1, tag_value=10.0, timestamp=1704110400.0), - ], - source=TagSourceType.Modeling, - ), - ] - - response = ModelingReadingsDataResponse(items=readings) - assert len(response.items) == 1 - assert response.items[0].tag_id == 1 - - def test_latest_readings_request(self): - """Test LatestReadingsRequest model.""" - request = LatestReadingsRequest(tag_ids=[1, 2, 3]) - assert request.tag_ids == [1, 2, 3] - - def test_latest_readings_response(self): - """Test LatestReadingsResponse model.""" - items = [ - LatestReadingsResponseItem( - tag_id=1, - tag_value=42.5, - tag_value_changed_at=datetime(2024, 1, 1, 12, 0, 0), - ), - ] - - response = LatestReadingsResponse(items=items) - assert len(response.items) == 1 - assert response.items[0].tag_id == 1 - assert response.items[0].tag_value == 42.5 + # Call the method + start_date = datetime(2024, 1, 1, 12, 0, 0) + end_date = datetime(2024, 1, 1, 13, 0, 0) + result = cvec.get_modeling_metrics( + start_at=start_date, + end_at=end_date, + ) -class TestModelingMethods: - """Test the modeling methods in the CVec class.""" + # Verify the result + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].id == 1 + assert result[0].name == "test_metric" - @patch('cvec.cvec.CVec._fetch_publishable_key') - @patch('cvec.cvec.CVec._login_with_supabase') - @patch('cvec.cvec.CVec._make_request') - def test_get_modeling_readings(self, mock_make_request, mock_login, mock_fetch_key): - """Test get_modeling_readings method.""" + # Verify the request was made correctly + mock_make_request.assert_called_once() + call_args = mock_make_request.call_args + assert call_args[0][0] == "GET" + assert call_args[0][1] == "/api/modeling/metrics" + assert call_args[1]["params"]["start_at"] == "2024-01-01T12:00:00" + assert call_args[1]["params"]["end_at"] == "2024-01-01T13:00:00" + + @patch("cvec.cvec.CVec._fetch_publishable_key") + @patch("cvec.cvec.CVec._login_with_supabase") + @patch("cvec.cvec.CVec._make_request") + def test_get_modeling_metrics_data( + self, mock_make_request: Mock, mock_login: Mock, mock_fetch_key: Mock + ) -> None: + """Test get_modeling_metrics_data method.""" # Mock the publishable key fetch mock_fetch_key.return_value = "test_publishable_key" - + # Mock the login method mock_login.return_value = None - + # Mock the response - mock_response = { - "items": [ - { - "tag_id": 1, - "data": [ - { - "tag_id": 1, - "tag_value": 10.0, - "timestamp": 1704110400.0, - } - ], - "source": "modeling", - } - ] - } + mock_response = [ + { + "name": "test_metric", + "time": "2024-01-01T12:00:00", + "value_double": 42.5, + "value_string": None, + } + ] mock_make_request.return_value = mock_response - + # Create CVec instance - cvec = CVec(host="http://test.com", api_key="cva_test12345678901234567890123456789012") - + cvec = CVec( + host="http://test.com", api_key="cva_test12345678901234567890123456789012" + ) + # Call the method start_date = datetime(2024, 1, 1, 12, 0, 0) end_date = datetime(2024, 1, 1, 13, 0, 0) - result = cvec.get_modeling_readings( - tag_ids=[1], + result = cvec.get_modeling_metrics_data( + names=["test_metric"], start_at=start_date, end_at=end_date, - desired_points=1000, ) - + # Verify the result - assert isinstance(result, ModelingReadingsDataResponse) - assert len(result.items) == 1 - assert result.items[0].tag_id == 1 - assert result.items[0].source == TagSourceType.Modeling - + assert isinstance(result, list) + assert len(result) == 1 + assert result[0].name == "test_metric" + assert result[0].value_double == 42.5 + # Verify the request was made correctly mock_make_request.assert_called_once() call_args = mock_make_request.call_args - assert call_args[0][0] == "POST" - assert call_args[0][1] == "/api/modeling/fetch_modeling_data" - assert call_args[1]["json"]["tag_ids"] == [1] - assert call_args[1]["json"]["start_date"] == "2024-01-01T12:00:00" - assert call_args[1]["json"]["end_date"] == "2024-01-01T13:00:00" - assert call_args[1]["json"]["desired_points"] == 1000 - - @patch('cvec.cvec.CVec._fetch_publishable_key') - @patch('cvec.cvec.CVec._login_with_supabase') - @patch('cvec.cvec.CVec._make_request') - def test_get_modeling_latest_readings(self, mock_make_request, mock_login, mock_fetch_key): - """Test get_modeling_latest_readings method.""" + assert call_args[0][0] == "GET" + assert call_args[0][1] == "/api/modeling/metrics/data" + assert call_args[1]["params"]["names"] == "test_metric" + assert call_args[1]["params"]["start_at"] == "2024-01-01T12:00:00" + assert call_args[1]["params"]["end_at"] == "2024-01-01T13:00:00" + + @patch("cvec.cvec.CVec._fetch_publishable_key") + @patch("cvec.cvec.CVec._login_with_supabase") + @patch("cvec.cvec.CVec._make_request") + def test_get_modeling_metrics_data_arrow( + self, mock_make_request: Mock, mock_login: Mock, mock_fetch_key: Mock + ) -> None: + """Test get_modeling_metrics_data_arrow method.""" # Mock the publishable key fetch mock_fetch_key.return_value = "test_publishable_key" - + # Mock the login method mock_login.return_value = None - - # Mock the response - mock_response = { - "items": [ - { - "tag_id": 1, - "tag_value": 42.5, - "tag_value_changed_at": "2024-01-01T12:00:00", - } - ] - } + + # Mock the response (Arrow data as bytes) + mock_response = b"fake_arrow_data" mock_make_request.return_value = mock_response - + # Create CVec instance - cvec = CVec(host="http://test.com", api_key="cva_test12345678901234567890123456789012") - + cvec = CVec( + host="http://test.com", api_key="cva_test12345678901234567890123456789012" + ) + # Call the method - result = cvec.get_modeling_latest_readings(tag_ids=[1]) - + start_date = datetime(2024, 1, 1, 12, 0, 0) + end_date = datetime(2024, 1, 1, 13, 0, 0) + result = cvec.get_modeling_metrics_data_arrow( + names=["test_metric"], + start_at=start_date, + end_at=end_date, + ) + # Verify the result - assert isinstance(result, LatestReadingsResponse) - assert len(result.items) == 1 - assert result.items[0].tag_id == 1 - assert result.items[0].tag_value == 42.5 - + assert isinstance(result, bytes) + assert result == b"fake_arrow_data" + # Verify the request was made correctly mock_make_request.assert_called_once() call_args = mock_make_request.call_args - assert call_args[0][0] == "POST" - assert call_args[0][1] == "/api/modeling/fetch_latest_readings" - assert call_args[1]["json"]["tag_ids"] == [1] + assert call_args[0][0] == "GET" + assert call_args[0][1] == "/api/modeling/metrics/data/arrow" + assert call_args[1]["params"]["names"] == "test_metric" + assert call_args[1]["params"]["start_at"] == "2024-01-01T12:00:00" + assert call_args[1]["params"]["end_at"] == "2024-01-01T13:00:00" if __name__ == "__main__": From 03a295c598887d11b84328dee01edb3a845e9c2a Mon Sep 17 00:00:00 2001 From: Amy Ni Date: Wed, 3 Sep 2025 14:36:27 -0400 Subject: [PATCH 3/4] chore: remove unused code --- src/cvec/models/__init__.py | 18 ------------ src/cvec/models/modeling.py | 55 ------------------------------------- 2 files changed, 73 deletions(-) delete mode 100644 src/cvec/models/modeling.py diff --git a/src/cvec/models/__init__.py b/src/cvec/models/__init__.py index 703120c..e3172fd 100644 --- a/src/cvec/models/__init__.py +++ b/src/cvec/models/__init__.py @@ -1,26 +1,8 @@ from .metric import Metric, MetricDataPoint -from .modeling import ( - FetchModelingReadingsRequest, - LatestReadingsRequest, - LatestReadingsResponse, - LatestReadingsResponseItem, - ModelingReadingsDataResponse, - ModelingReadingsGroup, - ModelingReadingModel, - TagSourceType, -) from .span import Span __all__ = [ "Metric", "MetricDataPoint", "Span", - "ModelingReadingsDataResponse", - "ModelingReadingsGroup", - "ModelingReadingModel", - "FetchModelingReadingsRequest", - "LatestReadingsRequest", - "LatestReadingsResponse", - "LatestReadingsResponseItem", - "TagSourceType", ] diff --git a/src/cvec/models/modeling.py b/src/cvec/models/modeling.py deleted file mode 100644 index dd27568..0000000 --- a/src/cvec/models/modeling.py +++ /dev/null @@ -1,55 +0,0 @@ -from datetime import datetime -from enum import Enum -from typing import List - -from pydantic import BaseModel - - -class TagSourceType(str, Enum): - Sensor = "sensor_readings" - Modeling = "modeling" - - -class BaseReadingModel(BaseModel): - tag_id: int - tag_value: float - timestamp: float - - -class ModelingReadingModel(BaseReadingModel): - pass - - -class BaseReadingsGroup(BaseModel): - tag_id: int - data: List[BaseReadingModel] - source: TagSourceType - - -class ModelingReadingsGroup(BaseReadingsGroup): - pass - - -class ModelingReadingsDataResponse(BaseModel): - items: List[ModelingReadingsGroup] - - -class FetchModelingReadingsRequest(BaseModel): - tag_ids: List[int] - start_date: datetime - end_date: datetime - desired_points: int = 10000 - - -class LatestReadingsRequest(BaseModel): - tag_ids: List[int] - - -class LatestReadingsResponseItem(BaseModel): - tag_id: int - tag_value: float - tag_value_changed_at: datetime - - -class LatestReadingsResponse(BaseModel): - items: List[LatestReadingsResponseItem] From 0b8a95774c47478e09ab2cc8e4f6cb1bf967df06 Mon Sep 17 00:00:00 2001 From: Amy Ni Date: Wed, 3 Sep 2025 14:59:49 -0400 Subject: [PATCH 4/4] chore: update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3111e68..66405bb 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ Add multiple metric data points to the database. Return a list of metrics that had at least one transition in the given [`start_at`, `end_at`) interval. All metrics are returned if no `start_at` and `end_at` are given. -## `get_modeling_metrics(start_at, end_at)` +## `get_modeling_metrics(?start_at, ?end_at)` Fetch modeling metrics from the modeling database. This method returns a list of available modeling metrics that had transitions in the specified time range. @@ -242,7 +242,7 @@ Fetch modeling metrics from the modeling database. This method returns a list of Returns a list of `Metric` objects containing modeling metrics. -## `get_modeling_metrics_data(names, start_at, end_at)` +## `get_modeling_metrics_data(?names, ?start_at, ?end_at)` Fetch actual data values from modeling metrics within a time range. This method returns the actual data points (values) for the specified modeling metrics, similar to `get_metric_data()` but for the modeling database. @@ -252,7 +252,7 @@ Fetch actual data values from modeling metrics within a time range. This method Returns a list of `MetricDataPoint` objects containing the actual data values. -## `get_modeling_metrics_data_arrow(names, start_at, end_at)` +## `get_modeling_metrics_data_arrow(?names, ?start_at, ?end_at)` Fetch actual data values from modeling metrics within a time range in Apache Arrow format. This method returns the actual data points (values) for the specified modeling metrics in Arrow IPC format, which is more efficient for large datasets.