First pass at duckdb data interface

pyproject.toml

@@ -34,7 +34,9 @@ dependencies = [
     "xlrd",
     "mypy-extensions",
     "pypubsub",
-    "tomlkit"
+    "tomlkit",
+    "duckdb",
+    "fsspec"
 ]
 dynamic = ["version"]
 

src/muse/data/example/default_new_input/agent_objectives.csv

@@ -0,0 +1,3 @@
+agent_id,objective,objective_data,objective_sort
+Agent1,LCOE,1,TRUE
+Agent2,LCOE,1,TRUE

src/muse/data/example/default_new_input/agent_pairs.csv

@@ -0,0 +1,2 @@
+id,new_agent_id,retrofit_agent_id,quantity
+A1,Agent1,Agent2,1

src/muse/data/example/default_new_input/agent_regions.csv

@@ -0,0 +1,2 @@
+agent_pair_id,region_id
+A1,R1

src/muse/data/example/default_new_input/agents.csv

@@ -0,0 +1,3 @@
+id,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule
+Agent1,New agent for A1,new,-1,inf,all,single
+Agent2,Retrofit agent for A1,retrofit,-1,inf,all,single

src/muse/data/example/default_new_input/assets.csv

@@ -0,0 +1,8 @@
+agent_id,process_id,region_id,year,capacity
+Agent2,gassupply1,R1,2020,15
+Agent2,gassupply1,R1,2025,15
+Agent2,gassupply1,R1,2030,7.5
+Agent2,gasCCGT,R1,2020,1
+Agent2,gasCCGT,R1,2025,1
+Agent2,gasboiler,R1,2020,10
+Agent2,gasboiler,R1,2025,5

src/muse/data/example/default_new_input/commodities.csv

@@ -0,0 +1,6 @@
+id,description,type,unit
+electricity,Electricity,energy,PJ
+gas,Gas,energy,PJ
+heat,Heat,energy,PJ
+wind,Wind,energy,PJ
+CO2f,Carbon dioxide,energy,kt

src/muse/data/example/default_new_input/commodity_costs.csv

@@ -0,0 +1,58 @@
+commodity_id,region_id,year,value
+electricity,R1,2010,14.81481472
+electricity,R1,2015,17.89814806
+electricity,R1,2020,19.5
+electricity,R1,2025,21.93518528
+electricity,R1,2030,26.50925917
+electricity,R1,2035,26.51851861
+electricity,R1,2040,23.85185194
+electricity,R1,2045,23.97222222
+electricity,R1,2050,24.06481472
+electricity,R1,2055,25.3425925
+electricity,R1,2060,25.53703694
+electricity,R1,2065,25.32407417
+electricity,R1,2070,23.36111111
+electricity,R1,2075,22.27777778
+electricity,R1,2080,22.25925917
+electricity,R1,2085,22.17592583
+electricity,R1,2090,22.03703694
+electricity,R1,2095,21.94444444
+electricity,R1,2100,21.39814806
+gas,R1,2010,6.6759
+gas,R1,2015,6.914325
+gas,R1,2020,7.15275
+gas,R1,2025,8.10645
+gas,R1,2030,9.06015
+gas,R1,2035,9.2191
+gas,R1,2040,9.37805
+gas,R1,2045,9.193829337
+gas,R1,2050,9.009608674
+gas,R1,2055,8.832625604
+gas,R1,2060,8.655642534
+gas,R1,2065,8.485612708
+gas,R1,2070,8.315582883
+gas,R1,2075,8.152233126
+gas,R1,2080,7.988883368
+gas,R1,2085,7.831951236
+gas,R1,2090,7.675019103
+gas,R1,2095,7.524252461
+gas,R1,2100,7.373485819
+CO2f,R1,2010,0
+CO2f,R1,2015,0.052913851
+CO2f,R1,2020,0.08314119
+CO2f,R1,2025,0.120069795
+CO2f,R1,2030,0.156998399
+CO2f,R1,2035,0.214877567
+CO2f,R1,2040,0.272756734
+CO2f,R1,2045,0.35394801
+CO2f,R1,2050,0.435139285
+CO2f,R1,2055,0.542365578
+CO2f,R1,2060,0.649591871
+CO2f,R1,2065,0.780892624
+CO2f,R1,2070,0.912193378
+CO2f,R1,2075,1.078321687
+CO2f,R1,2080,1.244449995
+CO2f,R1,2085,1.4253503
+CO2f,R1,2090,1.606250604
+CO2f,R1,2095,1.73877515
+CO2f,R1,2100,1.871299697

src/muse/data/example/default_new_input/commodity_trade.csv

@@ -0,0 +1 @@
+commodity_id,region_id,year,import,export

src/muse/data/example/default_new_input/demand.csv

@@ -0,0 +1,3 @@
+commodity_id,region_id,year,demand
+heat,R1,2020,10
+heat,R1,2050,30

src/muse/data/example/default_new_input/demand_slicing.csv

@@ -0,0 +1,13 @@
+commodity_id,region_id,year,timeslice_id,fraction
+heat,R1,2020,1,0.1
+heat,R1,2020,2,0.15
+heat,R1,2020,3,0.1
+heat,R1,2020,4,0.15
+heat,R1,2020,5,0.3
+heat,R1,2020,6,0.2
+heat,R1,2050,1,0.1
+heat,R1,2050,2,0.15
+heat,R1,2050,3,0.1
+heat,R1,2050,4,0.15
+heat,R1,2050,5,0.3
+heat,R1,2050,6,0.2

src/muse/data/example/default_new_input/process_availabilities.csv

@@ -0,0 +1,6 @@
+process_id,region_id,year,timeslice,lim_type,value
+gassupply1,,,ALL,UP,0.9
+gasCCGT,,,ALL,UP,0.9
+windturbine,,,ALL,UP,0.4
+gasboiler,,,ALL,UP,1
+heatpump,,,ALL,UP,1

src/muse/data/example/default_new_input/process_flows.csv

@@ -0,0 +1,12 @@
+process_id,commodity_id,region_id,year,flow
+gassupply1,gas,,,1
+gasCCGT,gas,,,-1.67
+gasCCGT,electricity,,,1
+gasCCGT,CO2f,,,91.67
+windturbine,wind,,,-1
+windturbine,electricity,,,1
+gasboiler,gas,,,-1.16
+gasboiler,heat,,,1
+gasboiler,CO2f,,,64.71
+heatpump,electricity,,,-0.4
+heatpump,heat,,,1

src/muse/data/example/default_new_input/process_parameters.csv

@@ -0,0 +1,6 @@
+process_id,region_id,year,cap_par,cap_exp,fix_par,fix_exp,var_par,var_exp,max_capacity_addition,max_capacity_growth,total_capacity_limit,life,scaling_size,efficiency,discount_rate
+gassupply1,,,0,1,0,1,2.55,1,5,1,60,35,0.00000189,86,0.1
+gasCCGT,,,23.78234399,1,0,1,0,1,2,1,60,35,0.00000189,86,0.1
+windturbine,,,36.30771182,1,0,1,0,1,2,1,60,25,0.00000189,86,0.1
+gasboiler,,,3.8,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1
+heatpump,,,8.866667,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1

src/muse/data/example/default_new_input/process_regions.csv

@@ -0,0 +1,6 @@
+process_id,region_id
+gassupply1,R1
+gasCCGT,R1
+windturbine,R1
+gasboiler,R1
+heatpump,R1

src/muse/data/example/default_new_input/processes.csv

@@ -0,0 +1,6 @@
+id,sector_id,type,fuel,end_use,level
+gassupply1,gas,energy,gas,gas,fixed
+gasCCGT,power,energy,gas,electricity,fixed
+windturbine,power,energy,wind,electricity,fixed
+gasboiler,residential,energy,gas,heat,fixed
+heatpump,residential,energy,electricity,heat,fixed

src/muse/data/example/default_new_input/regions.csv

@@ -0,0 +1,2 @@
+id,description
+R1,Region 1

src/muse/data/example/default_new_input/sectors.csv

@@ -0,0 +1,4 @@
+id,description
+gas,Gas sector
+power,Power sector
+residential,Residential sector

src/muse/data/example/default_new_input/timeslices.csv

@@ -0,0 +1,7 @@
+id,month,day,hour,fraction
+1,all-year,all-week,night,0.1667
+2,all-year,all-week,morning,0.1667
+3,all-year,all-week,afternoon,0.1667
+4,all-year,all-week,early-peak,0.1667
+5,all-year,all-week,late-peak,0.1667
+6,all-year,all-week,evening,0.1667

src/muse/examples.py

@@ -48,7 +48,8 @@ def example_data_dir() -> Path:
 
 def available_examples() -> list[str]:
     """List examples available in the examples folder."""
-    return [d.stem for d in example_data_dir().iterdir() if d.is_dir()]
+    # temporary skip for default_new_input as this is not yet working
+    return [d.stem for d in example_data_dir().iterdir()]
 
 
 def model(name: str = "default") -> MCA:
@@ -111,6 +112,8 @@ def copy_model(
         _copy_minimum_service(path)
     elif name.lower() == "trade":
         _copy_trade(path)
+    elif name.lower() == "default_new_input":
+        _copy_default_new_input(path)
     return path
 
 
@@ -287,6 +290,12 @@ def _copy_default(path: Path):
     copyfile(example_data_dir() / "default" / "settings.toml", path / "settings.toml")
 
 
+def _copy_default_new_input(path: Path):
+    from shutil import copytree
+
+    copytree(example_data_dir() / "default_new_input", path)
+
+
 def _copy_default_retro(path: Path):
     from shutil import copyfile, copytree