Generate FINE energy system model from database

ensysmod/api/endpoints/energy_models.py

@@ -1,10 +1,12 @@
 from typing import List, Union
 
 from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi.responses import FileResponse
 from sqlalchemy.orm import Session
 
 from ensysmod import schemas, model, crud
 from ensysmod.api import deps
+from ensysmod.core.fine_esm import generate_esm_from_model, optimize_esm
 
 router = APIRouter()
 
@@ -66,10 +68,10 @@ def update_model(model_id: int,
     Update a energy model.
     """
     # TODO Check if user has permission for model
-    model = crud.energy_model.get(db=db, id=model_id)
-    if model is None:
+    energy_model = crud.energy_model.get(db=db, id=model_id)
+    if energy_model is None:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"EnergyModel {model_id} not found!")
-    return crud.energy_model.update(db=db, db_obj=model, obj_in=request)
+    return crud.energy_model.update(db=db, db_obj=energy_model, obj_in=request)
 
 
 @router.delete("/{model_id}", response_model=schemas.EnergyModel)
@@ -81,3 +83,49 @@ def remove_model(model_id: int,
     """
     # TODO Check if user has permission for dataset
     return crud.energy_model.remove(db=db, id=model_id)
+
+
+@router.get("/{model_id}/esm", response_model=schemas.EnergyModel)
+def validate_model(model_id: int,
+                   db: Session = Depends(deps.get_db),
+                   current: model.User = Depends(deps.get_current_user)):
+    """
+    Create FINE energy system model from model.
+
+    Might take a while.
+    And return errors if dataset is not valid.
+    """
+    # TODO Check if user has permission for dataset
+    energy_model = crud.energy_model.get(db=db, id=model_id)
+    if energy_model is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"EnergyModel {model_id} not found!")
+
+    # TODO Check if user has permission for dataset
+
+    generate_esm_from_model(db=db, model=energy_model)
+    return energy_model
+
+
+@router.get("/{model_id}/optimize")
+def optimize_model(model_id: int,
+                   db: Session = Depends(deps.get_db),
+                   current: model.User = Depends(deps.get_current_user)):
+    """
+    Create FINE energy system model from model and optimizes it.
+
+    Might take a while.
+    And return errors if dataset is not valid.
+    """
+    # TODO Check if user has permission for dataset
+    energy_model = crud.energy_model.get(db=db, id=model_id)
+    if energy_model is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"EnergyModel {model_id} not found!")
+
+    # TODO Check if user has permission for dataset
+
+    esM = generate_esm_from_model(db=db, model=energy_model)
+    result_file_path = optimize_esm(esM=esM)
+
+    return FileResponse(result_file_path,
+                        media_type="application/vnd.openxmlformats-officedocument. spreadsheetml.sheet",
+                        filename=f"{energy_model.name}.xlsx")

ensysmod/core/fine_esm.py

@@ -0,0 +1,198 @@
+from datetime import datetime
+from typing import Any, Dict, List, Union
+
+import pandas as pd
+from FINE import EnergySystemModel, Storage, Sink, Transmission, Conversion, Source, writeOptimizationOutputToExcel
+from sqlalchemy.orm import Session
+
+from ensysmod import crud
+from ensysmod.model import EnergyModel, EnergyComponent, EnergySource, EnergySink, EnergyConversion, EnergyStorage, \
+    EnergyTransmission, EnergyModelParameter, EnergyModelParameterOperation, EnergyModelParameterAttribute
+
+# Dictionary that contains internal and fine model parameter names
+param_mapper: Dict[str, str] = {
+    'yearly_limit': 'yearlyLimit',
+}
+
+
+def generate_esm_from_model(db: Session, model: EnergyModel) -> EnergySystemModel:
+    """
+    Generate an ESM from a given EnergyModel.
+
+    :param db: Database session
+    :param model: EnergyModel
+    :return: ESM
+    """
+    regions = model.dataset.regions
+    region_ids = [region.id for region in regions]
+    commodities = model.dataset.commodities
+    esm_data = {
+        "hoursPerTimeStep": model.dataset.hours_per_time_step,
+        "numberOfTimeSteps": model.dataset.number_of_time_steps,
+        "costUnit": model.dataset.cost_unit,
+        "lengthUnit": model.dataset.length_unit,
+        "locations": set(region.name for region in regions),
+        "commodities": set(commodity.name for commodity in commodities),
+        "commodityUnitsDict": {commodity.name: commodity.unit for commodity in model.dataset.commodities},
+    }
+
+    esM = EnergySystemModel(verboseLogLevel=0, **esm_data)
+
+    # Add all sources
+    for source in model.dataset.sources:
+        add_source(esM, db, source, region_ids, model.parameters)
+
+    # Add all sinks
+    for sink in model.dataset.sinks:
+        add_sink(esM, db, sink, region_ids, model.parameters)
+
+    # Add all conversions
+    for conversion in model.dataset.conversions:
+        add_conversion(esM, db, conversion, region_ids, model.parameters)
+
+    # Add all storages
+    for storage in model.dataset.storages:
+        add_storage(esM, db, storage, region_ids, model.parameters)
+
+    # Add all transmissions
+    for transmission in model.dataset.transmissions:
+        add_transmission(esM, db, transmission, region_ids, model.parameters)
+
+    return esM
+
+
+def add_source(esM: EnergySystemModel, db: Session, source: EnergySource, region_ids: List[int],
+               custom_parameters: List[EnergyModelParameter]) -> None:
+    esm_source = component_to_dict(db, source.component, region_ids)
+    esm_source["commodity"] = source.commodity.name
+    if source.commodity_cost is not None:
+        esm_source["commodityCost"] = source.commodity_cost
+    esm_source = override_parameters(esm_source, custom_parameters)
+    esM.add(Source(esM=esM, **esm_source))
+
+
+def add_sink(esM: EnergySystemModel, db: Session, sink: EnergySink, region_ids: List[int],
+             custom_parameters: List[EnergyModelParameter]) -> None:
+    esm_sink = component_to_dict(db, sink.component, region_ids)
+    esm_sink["commodity"] = sink.commodity.name
+    esm_sink = override_parameters(esm_sink, custom_parameters)
+    esM.add(Sink(esM=esM, **esm_sink))
+
+
+def add_conversion(esM: EnergySystemModel, db: Session, conversion: EnergyConversion, region_ids: List[int],
+                   custom_parameters: List[EnergyModelParameter]) -> None:
+    esm_conversion = component_to_dict(db, conversion.component, region_ids)
+    esm_conversion["physicalUnit"] = conversion.commodity_unit.unit
+    esm_conversion["commodityConversionFactors"] = {x.commodity.name: x.conversion_factor for x in
+                                                    conversion.conversion_factors}
+    esm_conversion = override_parameters(esm_conversion, custom_parameters)
+    esM.add(Conversion(esM=esM, **esm_conversion))
+
+
+def add_storage(esM: EnergySystemModel, db: Session, storage: EnergyStorage, region_ids: List[int],
+                custom_parameters: List[EnergyModelParameter]) -> None:
+    esm_storage = component_to_dict(db, storage.component, region_ids)
+    esm_storage["commodity"] = storage.commodity.name
+    if storage.charge_efficiency is not None:
+        esm_storage["chargeEfficiency"] = storage.charge_efficiency
+    if storage.discharge_efficiency is not None:
+        esm_storage["dischargeEfficiency"] = storage.discharge_efficiency
+    if storage.self_discharge is not None:
+        esm_storage["selfDischarge"] = storage.self_discharge
+    if storage.cyclic_lifetime is not None:
+        esm_storage["cyclicLifetime"] = storage.cyclic_lifetime
+    if storage.charge_rate is not None:
+        esm_storage["chargeRate"] = storage.charge_rate
+    if storage.discharge_rate is not None:
+        esm_storage["dischargeRate"] = storage.discharge_rate
+    if storage.state_of_charge_min is not None:
+        esm_storage["stateOfChargeMin"] = storage.state_of_charge_min
+    if storage.state_of_charge_max is not None:
+        esm_storage["stateOfChargeMax"] = storage.state_of_charge_max
+    esm_storage = override_parameters(esm_storage, custom_parameters)
+    esM.add(Storage(esM=esM, **esm_storage))
+
+
+def add_transmission(esM: EnergySystemModel, db: Session, transmission: EnergyTransmission,
+                     region_ids: List[int], custom_parameters: List[EnergyModelParameter]) -> None:
+    esm_transmission = component_to_dict(db, transmission.component, region_ids)
+    esm_transmission["commodity"] = transmission.commodity.name
+    esm_transmission["distances"] = crud.energy_transmission_distance.get_dataframe(db, transmission.ref_component,
+                                                                                    region_ids=region_ids)
+    esm_transmission = override_parameters(esm_transmission, custom_parameters)
+    esM.add(Transmission(esM=esM, **esm_transmission))
+
+
+def component_to_dict(db: Session, component: EnergyComponent, region_ids: List[int]) -> Dict[str, Any]:
+    component_data = {
+        "name": component.name,
+        "hasCapacityVariable": component.capacity_variable,
+        "capacityVariableDomain": component.capacity_variable_domain.value.lower(),
+        "capacityPerPlantUnit": component.capacity_per_plant_unit,
+        "investPerCapacity": component.invest_per_capacity,
+        "opexPerCapacity": component.opex_per_capacity,
+        "interestRate": component.interest_rate,
+        "economicLifetime": component.economic_lifetime,
+    }
+    if component.shared_potential_id is not None:
+        component_data["sharedPotentialID"] = component.shared_potential_id
+
+    if crud.capacity_max.has_data(db, component_id=component.id, region_ids=region_ids):
+        component_data["capacityMax"] = df_or_s(crud.capacity_max.get_dataframe(db, component_id=component.id,
+                                                                                region_ids=region_ids))
+
+    if crud.capacity_fix.has_data(db, component_id=component.id, region_ids=region_ids):
+        component_data["capacityFix"] = df_or_s(crud.capacity_fix.get_dataframe(db, component_id=component.id,
+                                                                                region_ids=region_ids))
+
+    if crud.operation_rate_max.has_data(db, component_id=component.id, region_ids=region_ids):
+        component_data["operationRateMax"] = df_or_s(crud.operation_rate_max.get_dataframe(db,
+                                                                                           component_id=component.id,
+                                                                                           region_ids=region_ids))
+
+    if crud.operation_rate_fix.has_data(db, component_id=component.id, region_ids=region_ids):
+        component_data["operationRateFix"] = df_or_s(crud.operation_rate_fix.get_dataframe(db,
+                                                                                           component_id=component.id,
+                                                                                           region_ids=region_ids))
+
+    return component_data
+
+
+def override_parameters(component_dict: Dict, custom_parameters: List[EnergyModelParameter]) -> Dict:
+    for custom_parameter in custom_parameters:
+        if custom_parameter.component.name != component_dict["name"]:
+            continue
+        attribute_name = param_mapper[custom_parameter.attribute.name]
+        if custom_parameter.operation == EnergyModelParameterOperation.add:
+            component_dict[attribute_name] += custom_parameter.value
+        elif custom_parameter.operation == EnergyModelParameterOperation.multiply:
+            component_dict[attribute_name] *= custom_parameter.value
+        elif custom_parameter.operation == EnergyModelParameterOperation.set:
+            component_dict[attribute_name] = custom_parameter.value
+        else:
+            raise ValueError("Unknown operation: {}".format(custom_parameter.operation))
+        if custom_parameter.attribute == EnergyModelParameterAttribute.yearly_limit:
+            component_dict["commodityLimitID"] = "CO2"  # TODO: should be configurable
+    return component_dict
+
+
+def optimize_esm(esM: EnergySystemModel):
+    """
+    Optimize the energy system model.
+    """
+    esM.cluster(numberOfTypicalPeriods=7)
+    esM.optimize(timeSeriesAggregation=True, optimizationSpecs='OptimalityTol=1e-3 method=2 cuts=0', solver='gurobi')
+
+    time_str = datetime.now().strftime("%Y%m%d%H%M%S")
+    result_file_path = f"./tmp/result-{time_str}"
+    writeOptimizationOutputToExcel(esM=esM,
+                                   outputFileName=result_file_path,
+                                   optSumOutputLevel=2, optValOutputLevel=1)
+    return result_file_path + ".xlsx"
+
+
+def df_or_s(dataframe: pd.DataFrame) -> Union[pd.DataFrame, pd.Series]:
+    if dataframe.shape[0] == 1:
+        return dataframe.squeeze(axis=0)
+    else:
+        return dataframe

ensysmod/crud/base_depends_timeseries.py

@@ -54,16 +54,24 @@ def create(self, db: Session, *, obj_in: CreateSchemaType) -> ModelType:
 
         return super().create(db=db, obj_in=obj_in_dict)
 
-    def get_dataframe(self, db: Session, *, component_id: int, region_ids: List[id]) -> pd.DataFrame:
-        """
-        Get dataframe for component and multiple regions.
-        """
-        data = db.query(self.model) \
+    def get_multi_by_regions(self, db: Session, *,
+                             component_id: int, region_ids: List[id]) -> Optional[List[ModelType]]:
+        return db.query(self.model) \
             .filter(self.model.ref_component == component_id) \
             .filter(self.model.ref_region.in_(region_ids)) \
             .filter(or_(self.model.ref_region_to.is_(None), self.model.ref_region_to.in_(region_ids))) \
             .all()
 
+    def has_data(self, db: Session, *, component_id: int, region_ids: List[id]) -> bool:
+        result = self.get_multi_by_regions(db=db, component_id=component_id, region_ids=region_ids)
+        return result is not None and len(result) > 0
+
+    def get_dataframe(self, db: Session, *, component_id: int, region_ids: List[id]) -> pd.DataFrame:
+        """
+        Get dataframe for component and multiple regions.
+        """
+        data = self.get_multi_by_regions(db=db, component_id=component_id, region_ids=region_ids)
+
         matrix_mode = any(d.ref_region_to is not None for d in data)
 
         if matrix_mode and any(d.ref_region_to is None for d in data):

ensysmod/crud/energy_sink.py

@@ -1,7 +1,7 @@
 from sqlalchemy.orm import Session
 
-from ensysmod import crud
 from ensysmod.crud.base_depends_component import CRUDBaseDependsComponent
+from ensysmod.crud.energy_commodity import energy_commodity
 from ensysmod.model import EnergySink
 from ensysmod.schemas import EnergySinkCreate, EnergySinkUpdate
 
@@ -13,8 +13,8 @@ class CRUDEnergySink(CRUDBaseDependsComponent[EnergySink, EnergySinkCreate, Ener
     """
 
     def create(self, db: Session, *, obj_in: EnergySinkCreate) -> EnergySink:
-        commodity = crud.energy_commodity.get_by_dataset_and_name(db, name=obj_in.commodity,
-                                                                  dataset_id=obj_in.ref_dataset)
+        commodity = energy_commodity.get_by_dataset_and_name(db, name=obj_in.commodity,
+                                                             dataset_id=obj_in.ref_dataset)
         obj_in_dict = obj_in.dict()
         obj_in_dict['ref_commodity'] = commodity.id
         return super().create(db=db, obj_in=obj_in_dict)

ensysmod/crud/energy_source.py

@@ -1,7 +1,7 @@
 from sqlalchemy.orm import Session
 
-from ensysmod import crud
 from ensysmod.crud.base_depends_component import CRUDBaseDependsComponent
+from ensysmod.crud.energy_commodity import energy_commodity
 from ensysmod.model import EnergySource
 from ensysmod.schemas import EnergySourceCreate, EnergySourceUpdate
 
@@ -13,8 +13,8 @@ class CRUDEnergySource(CRUDBaseDependsComponent[EnergySource, EnergySourceCreate
     """
 
     def create(self, db: Session, *, obj_in: EnergySourceCreate) -> EnergySource:
-        commodity = crud.energy_commodity.get_by_dataset_and_name(db, name=obj_in.commodity,
-                                                                  dataset_id=obj_in.ref_dataset)
+        commodity = energy_commodity.get_by_dataset_and_name(db, name=obj_in.commodity,
+                                                             dataset_id=obj_in.ref_dataset)
         obj_in_dict = obj_in.dict()
         obj_in_dict['ref_commodity'] = commodity.id
         return super().create(db=db, obj_in=obj_in_dict)