From 5ee119971648db76b44d869e9f60587d6ece8860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20A=2E=20Rodr=C3=ADguez?= Date: Wed, 25 Feb 2026 17:25:09 +0100 Subject: [PATCH] feat(kpi): integrate KPI calculation into simulator worker task After create_output_esdl() writes InfluxDB profile references, call KpiManager.load_from_simulator() on the output ESDL so that profiles and KPIs coexist in the final result. System lifetime is read from workflow_config with a safe float() fallback. KpiValidationError is caught as a warning (simulation still returned without KPIs); unexpected failures are logged at ERROR with full traceback. Bump kpi-calculator dependency to >=0.3.0 / ==0.3.0 in requirements.txt. Add end-to-end integration test gated on INFLUXDB_HOSTNAME. Remove no-op test__hello_world cookiecutter artifact. --- dev-requirements.txt | 2 +- pyproject.toml | 4 +- requirements.txt | 4 +- src/simulator_worker/simulator_worker.py | 41 ++++++++++- unit_test/test_hello.py | 3 - unit_test/test_kpi_integration.py | 90 ++++++++++++++++++++++++ 6 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 unit_test/test_kpi_integration.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 36672d2..e9e9c50 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -281,7 +281,7 @@ tzdata==2025.2 # -c ..\..\requirements.txt # kombu # pandas -urllib3==2.5.0 +urllib3==2.6.3 # via # -c ..\..\requirements.txt # requests diff --git a/pyproject.toml b/pyproject.toml index 20e3cd4..4c38eba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,8 @@ dependencies = [ "omotes-sdk-python ~= 4.3.2", "omotes-simulator-core==0.0.28", "pyesdl==25.7", - "pandas ~= 2.2.2" + "pandas ~= 2.2.2", + "kpi-calculator>=0.3.0", ] [project.optional-dependencies] @@ -72,6 +73,7 @@ starting_version = "0.0.1" [tool.pytest.ini_options] addopts = "--cov=simulator_worker --cov-report html --cov-report term-missing --cov-fail-under 20" testpaths = ["unit_test"] +python_files = ["test_*.py"] [tool.coverage.run] source = ["src"] diff --git a/requirements.txt b/requirements.txt index 2f0d178..d09ace0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,6 +46,8 @@ influxdb==5.3.2 # via omotes-simulator-core kombu==5.5.4 # via celery +kpi-calculator==0.3.0 + # via simulator-worker (..\..\pyproject.toml) lxml==6.0.2 # via pyecore msgpack==1.1.2 @@ -121,7 +123,7 @@ tzdata==2025.2 # via # kombu # pandas -urllib3==2.5.0 +urllib3==2.6.3 # via requests vine==5.1.0 # via diff --git a/src/simulator_worker/simulator_worker.py b/src/simulator_worker/simulator_worker.py index 502d314..cda59d0 100644 --- a/src/simulator_worker/simulator_worker.py +++ b/src/simulator_worker/simulator_worker.py @@ -21,6 +21,7 @@ from uuid import uuid4 import dotenv +from esdl.esdl_handler import EnergySystemHandler from omotes_sdk.internal.orchestrator_worker_events.esdl_messages import EsdlMessage from omotes_sdk.internal.worker.worker import UpdateProgressHandler, initialize_worker from omotes_sdk.types import ProtobufDict @@ -36,7 +37,14 @@ from omotes_simulator_core.infrastructure.simulation_manager import SimulationManager from omotes_simulator_core.infrastructure.utils import pyesdl_from_string -from simulator_worker.utils import add_datetime_index, create_output_esdl +from kpicalculator import KpiManager +from kpicalculator.common.constants import DEFAULT_SYSTEM_LIFETIME_YEARS +from kpicalculator.exceptions import ValidationError as KpiValidationError + +from simulator_worker.utils import ( + add_datetime_index, + create_output_esdl, +) dotenv.load_dotenv() @@ -111,8 +119,39 @@ def simulator_worker_task( len(result_indexed.columns), result_indexed.shape, ) + output_esdl = create_output_esdl(input_esdl, result_indexed) + try: + system_lifetime = float( + workflow_config.get("system_lifetime", DEFAULT_SYSTEM_LIFETIME_YEARS) + ) + except (TypeError, ValueError): + system_lifetime = DEFAULT_SYSTEM_LIFETIME_YEARS + + try: + kpi_manager = KpiManager() + kpi_manager.load_from_simulator(result_indexed, esdl_string=output_esdl) + kpi_results = kpi_manager.calculate_all_kpis(system_lifetime=system_lifetime) + + esh = EnergySystemHandler() + esh.energy_system = kpi_manager.get_esdl_with_kpis(kpi_results, level="system") + output_esdl = esh.to_string() + logger.info("KPI calculation completed and added to output ESDL") + + except KpiValidationError as e: + # Expected failure: invalid ESDL structure or missing cost data. + # Simulation result is still valid — return it without KPIs. + logger.warning("KPI calculation skipped due to invalid input data: %s", e) + except Exception: + # Unexpected failure: log full traceback at ERROR so it is visible in + # monitoring, but keep the simulation result intact. + logger.error( + "KPI calculation failed unexpectedly. " + "Simulation will continue and return results without KPIs.\n%s", + traceback.format_exc(), + ) + # Write output_esdl to file for debugging # with open(f"result_{simulation_id}.esdl", "w") as file: # file.writelines(output_esdl) diff --git a/unit_test/test_hello.py b/unit_test/test_hello.py index 9c0cbe0..9aa1683 100644 --- a/unit_test/test_hello.py +++ b/unit_test/test_hello.py @@ -24,9 +24,6 @@ class TestHelloWorld(unittest.TestCase): - def test__hello_world(self) -> None: - print("Hello world!") - def test__add_datetime_index__happy_path(self) -> None: # Arrange df = pandas.DataFrame() diff --git a/unit_test/test_kpi_integration.py b/unit_test/test_kpi_integration.py new file mode 100644 index 0000000..203fae2 --- /dev/null +++ b/unit_test/test_kpi_integration.py @@ -0,0 +1,90 @@ +"""Test KPI integration with simulator-worker.""" + +import datetime +import os +from pathlib import Path +from unittest.mock import MagicMock + +import esdl +import pytest + +# Check if full simulator worker can be imported +SIMULATOR_AVAILABLE = False +try: + from simulator_worker.simulator_worker import simulator_worker_task + from omotes_simulator_core.infrastructure.utils import pyesdl_from_string + + SIMULATOR_AVAILABLE = True +except ImportError: + simulator_worker_task = None # type: ignore[assignment, misc] + pyesdl_from_string = None # type: ignore[assignment, misc] + + +@pytest.mark.skipif( + not SIMULATOR_AVAILABLE or os.getenv("INFLUXDB_HOSTNAME") is None, + reason="simulator-worker or InfluxDB not available" +) +class TestKPIEndToEndIntegration: + """Integration tests for end-to-end KPI calculation in simulator workflow.""" + + def test_kpis_included_in_output_esdl(self) -> None: + """Test that KPIs are calculated and included in output ESDL.""" + # Load test ESDL + test_esdl_path = Path(__file__).parent.parent / "testdata" / "test1.esdl" + with open(test_esdl_path, "r") as f: + input_esdl = f.read() + + # Configure workflow - use Unix timestamps (seconds since epoch) + start_time = datetime.datetime(2019, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) + end_time = datetime.datetime(2019, 1, 1, 2, 0, tzinfo=datetime.timezone.utc) + + workflow_config: dict[str, list[float] | float | str | bool] = { + "timestep": 3600.0, # 1 hour in seconds + "start_time": start_time.timestamp(), # Unix timestamp + "end_time": end_time.timestamp(), # Unix timestamp + "system_lifetime": 30.0, + } + + # Mock progress handler + mock_progress = MagicMock() + + # Run simulation with KPI calculation + output_esdl, _ = simulator_worker_task( + input_esdl, workflow_config, mock_progress, "simulator" + ) + + # Verify output is not None + assert output_esdl is not None + assert len(output_esdl) > 0 + + # Parse output ESDL + esh = pyesdl_from_string(output_esdl) + energy_system = esh.energy_system + + assert hasattr(energy_system, "KPIs"), "Energy system should have KPIs attribute" + kpis = energy_system.KPIs + assert kpis is not None, "KPIs should be calculated and present in output ESDL" + + assert hasattr(kpis, "kpi"), "KPIs should have kpi collection" + kpi_list = list(kpis.kpi) + assert len(kpi_list) > 0, "At least one KPI should be present in output ESDL" + + for kpi in kpi_list: + assert isinstance(kpi, esdl.DistributionKPI), f"Expected DistributionKPI, got {type(kpi)}" + assert kpi.name, "KPI should have a non-empty name" + assert kpi.distribution is not None, f"KPI '{kpi.name}' has no distribution" + items = list(kpi.distribution.stringItem) + assert len(items) > 0, f"KPI '{kpi.name}' distribution has no items" + for item in items: + assert item.value is not None, f"KPI '{kpi.name}' item '{item.label}' has no value" + + # InfluxDB profile references must survive the KPI enrichment step + all_ports = [ + port + for asset in energy_system.eAllContents() + if isinstance(asset, esdl.Asset) + for port in asset.port + ] + assert any( + len(port.profile) > 0 for port in all_ports + ), "Output ESDL should retain InfluxDB profile references from simulation results"