From 1504dc662f7a702323853e3bacfbee0a76e044ea Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 30 Jan 2025 11:17:57 +0000 Subject: [PATCH 01/14] Move broadcast_techs out of quantities module --- src/muse/demand_share.py | 23 ++++++++++++++--------- src/muse/quantities.py | 29 ++++++++--------------------- src/muse/sectors/sector.py | 12 ++++++------ 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/src/muse/demand_share.py b/src/muse/demand_share.py index 40397bf3f..f838c05b0 100644 --- a/src/muse/demand_share.py +++ b/src/muse/demand_share.py @@ -246,13 +246,13 @@ def new_and_retro( from muse.commodities import is_enduse from muse.quantities import maximum_production - from muse.utilities import agent_concatenation, reduce_assets + from muse.utilities import agent_concatenation, broadcast_techs, reduce_assets current_year, investment_year = map(int, market.year.values) def decommissioning(capacity): return decommissioning_demand( - technologies=technologies, + technologies=technodata, capacity=capacity.interp( year=[current_year, investment_year], kwargs={"fill_value": 0.0} ), @@ -263,10 +263,13 @@ def decommissioning(capacity): year=[current_year, investment_year], kwargs={"fill_value": 0.0} ) + # Select technodata for assets + technodata = broadcast_techs(technologies, capacity, installed_as_year=True) + demands = new_and_retro_demands( capacity, market, - technologies, + technodata, timeslice_level=timeslice_level, ) @@ -284,7 +287,6 @@ def decommissioning(capacity): id_to_share: MutableMapping[Hashable, xr.DataArray] = {} for region in demands.region.values: - regional_techs = technologies.sel(region=region) retro_capacity: MutableMapping[Hashable, xr.DataArray] = { agent.uuid: agent.assets.capacity for agent in agents @@ -330,7 +332,7 @@ def decommissioning(capacity): demands.new.sel(region=region), partial( maximum_production, - technologies=regional_techs, + technologies=technodata, year=current_year, timeslice_level=timeslice_level, ), @@ -372,13 +374,13 @@ def standard_demand( from muse.commodities import is_enduse from muse.quantities import maximum_production - from muse.utilities import agent_concatenation, reduce_assets + from muse.utilities import agent_concatenation, broadcast_techs, reduce_assets current_year, investment_year = map(int, market.year.values) def decommissioning(capacity): return decommissioning_demand( - technologies=technologies, + technologies=technodata, capacity=capacity.interp( year=[current_year, investment_year], kwargs={"fill_value": 0.0} ), @@ -395,11 +397,14 @@ def decommissioning(capacity): year=[current_year, investment_year], kwargs={"fill_value": 0.0} ) + # Select technodata for assets + technodata = broadcast_techs(technologies, capacity, installed_as_year=True) + # Calculate new and retrofit demands demands = new_and_retro_demands( capacity=capacity, market=market, - technologies=technologies, + technologies=technodata, timeslice_level=timeslice_level, ) @@ -434,7 +439,7 @@ def decommissioning(capacity): demands.new.sel(region=region), partial( maximum_production, - technologies=technologies.sel(region=region), + technologies=technodata, year=current_year, timeslice_level=timeslice_level, ), diff --git a/src/muse/quantities.py b/src/muse/quantities.py index 8878d0e85..90670351f 100644 --- a/src/muse/quantities.py +++ b/src/muse/quantities.py @@ -114,19 +114,15 @@ def emission( A data array containing emissions (and only emissions). """ from muse.commodities import is_pollutant - from muse.utilities import broadcast_techs - - assert "asset" in production.dims # Calculate the production amplitude of each asset - techs = broadcast_techs(technologies, production, installed_as_year=True) - prod_amplitude = production_amplitude(production, techs) + prod_amplitude = production_amplitude(production, technologies) # Calculate the production of environmental pollutants # = prod_amplitude * fixed_outputs - envs = is_pollutant(techs.comm_usage) + envs = is_pollutant(technologies.comm_usage) envs_production = prod_amplitude * broadcast_timeslice( - techs.sel(commodity=envs).fixed_outputs, level=timeslice_level + technologies.sel(commodity=envs).fixed_outputs, level=timeslice_level ) return envs_production @@ -243,18 +239,14 @@ def maximum_production( filters and the set of technologies in `capacity`. """ from muse.commodities import is_enduse - from muse.utilities import broadcast_techs, filter_input + from muse.utilities import filter_input capa = filter_input( capacity, **{k: v for k, v in filters.items() if k in capacity.dims} ) - btechs = broadcast_techs( - technologies[["fixed_outputs", "utilization_factor"]], - capa, - installed_as_year=True, - ) + ftechs = filter_input( - btechs, **{k: v for k, v in filters.items() if k in btechs.dims} + technologies, **{k: v for k, v in filters.items() if k in technologies.dims} ) result = ( broadcast_timeslice(capa, level=timeslice_level) @@ -359,7 +351,7 @@ def minimum_production( the filters and the set of technologies in `capacity`. """ from muse.commodities import is_enduse - from muse.utilities import broadcast_techs, filter_input + from muse.utilities import filter_input capa = filter_input( capacity, **{k: v for k, v in filters.items() if k in capacity.dims} @@ -368,13 +360,8 @@ def minimum_production( if "minimum_service_factor" not in technologies: return broadcast_timeslice(xr.zeros_like(capa), level=timeslice_level) - btechs = broadcast_techs( - technologies[["fixed_outputs", "minimum_service_factor"]], - capa, - installed_as_year=True, - ) ftechs = filter_input( - btechs, **{k: v for k, v in filters.items() if k in btechs.dims} + technologies, **{k: v for k, v in filters.items() if k in technologies.dims} ) result = ( broadcast_timeslice(capa, level=timeslice_level) diff --git a/src/muse/sectors/sector.py b/src/muse/sectors/sector.py index 71a8e0de0..a1e9d2d28 100644 --- a/src/muse/sectors/sector.py +++ b/src/muse/sectors/sector.py @@ -296,26 +296,26 @@ def market_variables(self, market: xr.Dataset, technologies: xr.Dataset) -> Any: years = market.year.values capacity = self.capacity.interp(year=years, **self.interpolation) + # Select technology data for each asset + # Each asset uses the technology data from the year it was installed + technodata = broadcast_techs(technologies, capacity, installed_as_year=True) + # Calculate supply supply = self.supply_prod( market=market, capacity=capacity, - technologies=technologies, + technologies=technodata, timeslice_level=self.timeslice_level, ) # Calculate consumption consume = consumption( - technologies, + technologies=technodata, production=supply, prices=market.prices, timeslice_level=self.timeslice_level, ) - # Select technology data for each asset - # Each asset uses the technology data from the year it was installed - technodata = broadcast_techs(technologies, supply, installed_as_year=True) - # Calculate LCOE # We select data for the second year, which corresponds to the investment year lcoe = levelized_cost_of_energy( From ff906c752be6582a9352c2851867c95b9a230c37 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 30 Jan 2025 11:53:15 +0000 Subject: [PATCH 02/14] Update _inner_split --- src/muse/demand_share.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/muse/demand_share.py b/src/muse/demand_share.py index f838c05b0..1718cfdd9 100644 --- a/src/muse/demand_share.py +++ b/src/muse/demand_share.py @@ -250,9 +250,9 @@ def new_and_retro( current_year, investment_year = map(int, market.year.values) - def decommissioning(capacity): + def decommissioning(capacity, technologies): return decommissioning_demand( - technologies=technodata, + technologies=technologies, capacity=capacity.interp( year=[current_year, investment_year], kwargs={"fill_value": 0.0} ), @@ -292,13 +292,15 @@ def decommissioning(capacity): for agent in agents if agent.category == "retrofit" and agent.region == region } - + retro_technodata: MutableMapping[Hashable, xr.Dataset] = { + agent_uuid: technodata.sel(asset=retro_capacity[agent_uuid].asset) + for agent_uuid in retro_capacity.keys() + } name_to_id = { (agent.name, agent.region): agent.uuid for agent in agents if agent.category == "retrofit" and agent.region == region } - id_to_rquantity = { agent.uuid: (agent.name, agent.region, agent.quantity) for agent in agents @@ -307,6 +309,7 @@ def decommissioning(capacity): retro_demands: MutableMapping[Hashable, xr.DataArray] = _inner_split( retro_capacity, + retro_technodata, demands.retrofit.sel(region=region), decommissioning, id_to_rquantity, @@ -321,7 +324,10 @@ def decommissioning(capacity): for agent in agents if agent.category != "retrofit" and agent.region == region } - + new_technodata: MutableMapping[Hashable, xr.Dataset] = { + agent_uuid: technodata.sel(asset=new_capacity[agent_uuid].asset) + for agent_uuid in new_capacity.keys() + } id_to_nquantity = { agent.uuid: (agent.name, agent.region, agent.quantity) for agent in agents @@ -329,10 +335,10 @@ def decommissioning(capacity): } new_demands = _inner_split( new_capacity, + new_technodata, demands.new.sel(region=region), partial( maximum_production, - technologies=technodata, year=current_year, timeslice_level=timeslice_level, ), @@ -378,9 +384,9 @@ def standard_demand( current_year, investment_year = map(int, market.year.values) - def decommissioning(capacity): + def decommissioning(capacity, technologies): return decommissioning_demand( - technologies=technodata, + technologies=technologies, capacity=capacity.interp( year=[current_year, investment_year], kwargs={"fill_value": 0.0} ), @@ -421,6 +427,10 @@ def decommissioning(capacity): for agent in agents if agent.region == region } + current_technodata: MutableMapping[Hashable, xr.Dataset] = { + agent_uuid: technodata.sel(asset=current_capacity[agent_uuid].asset) + for agent_uuid in current_capacity.keys() + } # Split demands between agents id_to_quantity = { @@ -430,16 +440,17 @@ def decommissioning(capacity): } retro_demands: MutableMapping[Hashable, xr.DataArray] = _inner_split( current_capacity, + current_technodata, demands.retrofit.sel(region=region), decommissioning, id_to_quantity, ) new_demands = _inner_split( current_capacity, + current_technodata, demands.new.sel(region=region), partial( maximum_production, - technologies=technodata, year=current_year, timeslice_level=timeslice_level, ), @@ -495,6 +506,7 @@ def unmet_forecasted_demand( def _inner_split( assets: Mapping[Hashable, xr.DataArray], + technologies: Mapping[Hashable, xr.DataSet], demand: xr.DataArray, method: Callable, quantity: Mapping, @@ -508,7 +520,7 @@ def _inner_split( # Find decrease in capacity production by each asset over time shares: Mapping[Hashable, xr.DataArray] = { - key: method(capacity=capacity) + key: method(capacity=capacity, technologies=technologies[key]) .groupby("technology") .sum("asset") .rename(technology="asset") From 8ff2aea0a503abd6344301a196d438f3b6473345 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 30 Jan 2025 12:22:15 +0000 Subject: [PATCH 03/14] Fix trade model --- src/muse/demand_share.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/muse/demand_share.py b/src/muse/demand_share.py index 1718cfdd9..e7047bb38 100644 --- a/src/muse/demand_share.py +++ b/src/muse/demand_share.py @@ -477,7 +477,7 @@ def unmet_forecasted_demand( ) -> xr.DataArray: """Forecast demand that cannot be serviced by non-decommissioned current assets.""" from muse.commodities import is_enduse - from muse.utilities import reduce_assets + from muse.utilities import broadcast_techs, reduce_assets current_year, investment_year = map(int, market.year.values) @@ -493,11 +493,14 @@ def unmet_forecasted_demand( future_market = smarket.sel(year=investment_year, drop=True) future_capacity = capacity.sel(year=investment_year) + # Select technology data for assets + techs = broadcast_techs(technologies, capacity, installed_as_year=True) + # Calculate unmet demand result = unmet_demand( market=future_market, capacity=future_capacity, - technologies=technologies, + technologies=techs, timeslice_level=timeslice_level, ) assert "year" not in result.dims From 84c02d5130d6d42c12e15121594b2a085ac6a030 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 30 Jan 2025 13:03:28 +0000 Subject: [PATCH 04/14] Fix constraints tests --- tests/test_constraints.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_constraints.py b/tests/test_constraints.py index 4293262c4..c9ad0f13e 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -58,9 +58,10 @@ def capacity(assets): @fixture def market_demand(assets, technologies): from muse.quantities import maximum_production + from muse.utilities import broadcast_techs return 0.8 * maximum_production( - technologies, + broadcast_techs(technologies, assets), assets.capacity, ).sel(year=INVESTMENT_YEAR).groupby("technology").sum("asset").rename( technology="asset" From 7b8e8f01769ff6829e8c03e763f5dec95f457db5 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 30 Jan 2025 13:18:43 +0000 Subject: [PATCH 05/14] Fix demand_share tests --- tests/test_demand_share.py | 59 ++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/tests/test_demand_share.py b/tests/test_demand_share.py index e210dbb0d..125708405 100644 --- a/tests/test_demand_share.py +++ b/tests/test_demand_share.py @@ -55,13 +55,18 @@ def test_fixtures(_capacity, _market, _technologies): def test_new_retro_split_zero_unmet(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands + from muse.utilities import broadcast_techs + _technologies = broadcast_techs(_technologies, _capacity) share = new_and_retro_demands(_capacity, _market, _technologies) assert (share == 0).all() def test_new_retro_split_zero_consumption_increase(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands + from muse.utilities import broadcast_techs + + _technologies = broadcast_techs(_technologies, _capacity) _market.consumption.loc[{"year": INVESTMENT_YEAR}] = _market.consumption.sel( year=CURRENT_YEAR @@ -92,6 +97,9 @@ def test_new_retro_split_zero_consumption_increase(_capacity, _market, _technolo def test_new_retro_split_zero_new_unmet(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands + from muse.utilities import broadcast_techs + + _technologies = broadcast_techs(_technologies, _capacity) _market.consumption.loc[{"year": INVESTMENT_YEAR}] = _market.supply.sel( year=CURRENT_YEAR, drop=True @@ -127,6 +135,9 @@ def test_new_retro_split_zero_new_unmet(_capacity, _market, _technologies): def test_new_retro_accounting_identity(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands from muse.quantities import maximum_production + from muse.utilities import broadcast_techs + + _technologies = broadcast_techs(_technologies, _capacity) share = new_and_retro_demands(_capacity, _market, _technologies) assert (share >= 0).all() @@ -155,12 +166,13 @@ def test_new_retro_accounting_identity(_capacity, _market, _technologies): def test_demand_split(_capacity, _market, _technologies): from muse.commodities import is_enduse from muse.demand_share import _inner_split as inner_split + from muse.utilities import broadcast_techs - def method(capacity): + def method(capacity, technologies): from muse.demand_share import decommissioning_demand return decommissioning_demand( - _technologies.sel(region="USA"), + technologies, capacity, ) @@ -168,8 +180,10 @@ def method(capacity): year=INVESTMENT_YEAR, region="USA", drop=True ).where(is_enduse(_technologies.comm_usage.sel(commodity=_market.commodity))) agents = dict(scully=_capacity, mulder=_capacity) + _technologies = broadcast_techs(_technologies, _capacity) + technodata = dict(scully=_technologies, mulder=_technologies) quantity = dict(scully=("scully", "USA", 0.3), mulder=("mulder", "USA", 0.7)) - share = inner_split(agents, demand, method, quantity) + share = inner_split(agents, technodata, demand, method, quantity) enduse = is_enduse(_technologies.comm_usage) assert (share["scully"].sel(commodity=~enduse) == 0).all() @@ -189,12 +203,13 @@ def test_demand_split_zero_share(_capacity, _market, _technologies): """See issue SgiModel/StarMuse#688.""" from muse.commodities import is_enduse from muse.demand_share import _inner_split as inner_split + from muse.utilities import broadcast_techs - def method(capacity): + def method(capacity, technologies): from muse.demand_share import decommissioning_demand return 0 * decommissioning_demand( - _technologies.sel(region="USA"), + technologies, capacity, ) @@ -202,8 +217,10 @@ def method(capacity): year=INVESTMENT_YEAR, region="USA", drop=True ).where(is_enduse(_technologies.comm_usage.sel(commodity=_market.commodity))) agents = dict(scully=0.3 * _capacity, mulder=0.7 * _capacity) + _technologies = broadcast_techs(_technologies, _capacity) + technodata = dict(scully=_technologies, mulder=_technologies) quantity = dict(scully=("scully", "USA", 1), mulder=("mulder", "USA", 1)) - share = inner_split(agents, demand, method, quantity) + share = inner_split(agents, technodata, demand, method, quantity) enduse = is_enduse(_technologies.comm_usage) assert (share["scully"].sel(commodity=~enduse) == 0).all() @@ -227,12 +244,17 @@ def test_new_retro_demand_share(_technologies, market, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import new_and_retro + from muse.utilities import broadcast_techs asia_stock = stock.where(stock.region == "ASEAN", drop=True) usa_stock = stock.where(stock.region == "USA", drop=True) - asia_market = _matching_market(_technologies, asia_stock.capacity) - usa_market = _matching_market(_technologies, usa_stock.capacity) + asia_market = _matching_market( + broadcast_techs(_technologies, asia_stock), asia_stock.capacity + ) + usa_market = _matching_market( + broadcast_techs(_technologies, usa_stock), usa_stock.capacity + ) market = xr.concat((asia_market, usa_market), dim="region") market.consumption.loc[{"year": 2030}] *= 2 @@ -281,12 +303,17 @@ def test_standard_demand_share(_technologies, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import standard_demand from muse.errors import RetrofitAgentInStandardDemandShare + from muse.utilities import broadcast_techs asia_stock = stock.where(stock.region == "ASEAN", drop=True) usa_stock = stock.where(stock.region == "USA", drop=True) - asia_market = _matching_market(_technologies, asia_stock.capacity) - usa_market = _matching_market(_technologies, usa_stock.capacity) + asia_market = _matching_market( + broadcast_techs(_technologies, asia_stock), asia_stock.capacity + ) + usa_market = _matching_market( + broadcast_techs(_technologies, usa_stock), usa_stock.capacity + ) market = xr.concat((asia_market, usa_market), dim="region") market.consumption.loc[{"year": 2030}] *= 2 @@ -332,12 +359,17 @@ def test_unmet_forecast_demand(_technologies, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import unmet_forecasted_demand + from muse.utilities import broadcast_techs asia_stock = stock.where(stock.region == "ASEAN", drop=True) usa_stock = stock.where(stock.region == "USA", drop=True) - asia_market = _matching_market(_technologies, asia_stock.capacity) - usa_market = _matching_market(_technologies, usa_stock.capacity) + asia_market = _matching_market( + broadcast_techs(_technologies, asia_stock), asia_stock.capacity + ) + usa_market = _matching_market( + broadcast_techs(_technologies, usa_stock), usa_stock.capacity + ) market = xr.concat((asia_market, usa_market), dim="region") # spoof some agents @@ -387,6 +419,9 @@ class Agent: def test_decommissioning_demand(_technologies, _capacity, timeslice): from muse.commodities import is_enduse from muse.demand_share import decommissioning_demand + from muse.utilities import broadcast_techs + + _technologies = broadcast_techs(_technologies, _capacity) _capacity.loc[{"year": CURRENT_YEAR}] = current = 1.3 _capacity.loc[{"year": INVESTMENT_YEAR}] = forecast = 1.0 From 3fc7dcdcfb223b75d8c558607af9d8ca7168cf38 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 30 Jan 2025 13:36:30 +0000 Subject: [PATCH 06/14] Fix quantities tests --- tests/test_quantities.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/test_quantities.py b/tests/test_quantities.py index 2a5d0e8d6..304a0daf0 100644 --- a/tests/test_quantities.py +++ b/tests/test_quantities.py @@ -3,18 +3,23 @@ from pytest import approx, fixture, mark +@fixture +def technologies(technologies, capacity, timeslice): + from muse.utilities import broadcast_techs + + return broadcast_techs(technologies, capacity) + + @fixture def production( technologies: xr.Dataset, capacity: xr.DataArray, timeslice ) -> xr.DataArray: from muse.timeslices import broadcast_timeslice, distribute_timeslice - from muse.utilities import broadcast_techs - techs = broadcast_techs(technologies, capacity) return ( broadcast_timeslice(capacity) - * distribute_timeslice(techs.fixed_outputs) - * broadcast_timeslice(techs.utilization_factor) + * distribute_timeslice(technologies.fixed_outputs) + * broadcast_timeslice(technologies.utilization_factor) ) @@ -36,7 +41,7 @@ def test_consumption(technologies, production, market): def test_production_aggregate_asset_view( - capacity: xr.DataArray, technologies: xr.Dataset + technologies: xr.Dataset, capacity: xr.DataArray ): """Production when capacity has format of agent.sector. @@ -85,7 +90,7 @@ def test_production_aggregate_asset_view( @mark.xfail def test_production_agent_asset_view( - capacity: xr.DataArray, technologies: xr.Dataset, timeslice + technologies: xr.Dataset, capacity: xr.DataArray, timeslice ): """Production when capacity has format of agent.assets.capacity. @@ -95,7 +100,7 @@ def test_production_agent_asset_view( from muse.utilities import coords_to_multiindex, reduce_assets capacity = coords_to_multiindex(reduce_assets(capacity)).unstack("asset").fillna(0) - test_production_aggregate_asset_view(capacity, technologies) + test_production_aggregate_asset_view(technologies, capacity) def test_capacity_in_use(production: xr.DataArray, technologies: xr.Dataset): @@ -172,7 +177,7 @@ def test_supply_single_region(technologies, capacity, production, timeslice): # Select data for a single region region = "USA" - technologies = technologies.sel(region=region) + technologies = technologies.where(technologies.region == region, drop=True) capacity = capacity.where(capacity.region == region, drop=True) production = production.where(production.region == region, drop=True) From 1c572f1ec0b7171b12cfc7fac714be599ad55f2c Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Thu, 30 Jan 2025 13:44:00 +0000 Subject: [PATCH 07/14] Fix trade tests --- src/muse/examples.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/muse/examples.py b/src/muse/examples.py index 963e9e67e..f5a9949cb 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -264,16 +264,14 @@ def matching_market(sector: str, model: str = "default") -> xr.Dataset: from muse.examples import sector as load_sector from muse.quantities import consumption, maximum_production from muse.sectors import Sector - from muse.utilities import agent_concatenation + from muse.utilities import agent_concatenation, broadcast_techs loaded_sector = cast(Sector, load_sector(sector, model)) assets = agent_concatenation({u.uuid: u.assets for u in list(loaded_sector.agents)}) market = xr.Dataset() - production = cast( - xr.DataArray, - maximum_production(loaded_sector.technologies, assets.capacity), - ) + techs = broadcast_techs(loaded_sector.technologies, assets.capacity) + production = maximum_production(techs, assets.capacity) market["supply"] = production.sum("asset") if "dst_region" in market.dims: market = market.rename(dst_region="region") From d72f7c95a78f42061ddea5b789f33e1d0326f5d9 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 4 Feb 2025 10:09:39 +0000 Subject: [PATCH 08/14] Fix adhoc model --- src/muse/investments.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/muse/investments.py b/src/muse/investments.py index 948039ef8..055c6811a 100644 --- a/src/muse/investments.py +++ b/src/muse/investments.py @@ -225,14 +225,16 @@ def adhoc_match_demand( ) -> xr.DataArray: from muse.demand_matching import demand_matching from muse.quantities import capacity_in_use, maximum_production + from muse.utilities import broadcast_techs assert "year" not in technologies.dims demand = next(c for c in constraints if c.name == "demand").b max_capacity = next(c for c in constraints if c.name == "max capacity expansion").b + techs = broadcast_techs(technologies, max_capacity) max_prod = maximum_production( - technologies, + techs, max_capacity, technology=costs.replacement, commodity=demand.commodity, From dc616ae991905ffee37837e50746a1e810ad1ff4 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 4 Feb 2025 10:24:38 +0000 Subject: [PATCH 09/14] Rename broadcast_techs to broadcast_over_assets --- src/muse/__main__.py | 4 +-- src/muse/agents/agent.py | 1 - src/muse/demand_share.py | 12 ++++----- src/muse/examples.py | 4 +-- src/muse/investments.py | 4 +-- src/muse/quantities.py | 17 ++++++------- src/muse/sectors/sector.py | 6 +++-- src/muse/utilities.py | 14 +++++------ tests/conftest.py | 4 +-- tests/test_constraints.py | 4 +-- tests/test_costs.py | 8 +++--- tests/test_demand_share.py | 50 +++++++++++++++++++------------------- tests/test_objectives.py | 8 +++--- tests/test_quantities.py | 8 +++--- tests/test_utilities.py | 10 ++++---- 15 files changed, 75 insertions(+), 79 deletions(-) diff --git a/src/muse/__main__.py b/src/muse/__main__.py index cd0ad90f5..5286dc93a 100644 --- a/src/muse/__main__.py +++ b/src/muse/__main__.py @@ -114,8 +114,8 @@ def patched_broadcast_compat_data(self, other): "(e.g. a capacity dataset with an 'asset' dimension) with a fully explicit " "technology dataset (e.g. a technology dataset with 'region' and " "'technology' dimensions). " - "Please use `broadcast_techs` on the latter object before performing this " - "operation." + "Please use `broadcast_over_assets` on the latter object before performing " + "this operation." ) return self_data, other_data, dims diff --git a/src/muse/agents/agent.py b/src/muse/agents/agent.py index a4e05f160..159f39e77 100644 --- a/src/muse/agents/agent.py +++ b/src/muse/agents/agent.py @@ -257,7 +257,6 @@ def asset_housekeeping(self): - remove empty assets - remove years prior to current - - interpolate current year and forecasted year """ # TODO: move this into search and make sure filters, demand_share and # what not use assets from search. That would remove another bit of diff --git a/src/muse/demand_share.py b/src/muse/demand_share.py index e7047bb38..17b0be3b4 100644 --- a/src/muse/demand_share.py +++ b/src/muse/demand_share.py @@ -246,7 +246,7 @@ def new_and_retro( from muse.commodities import is_enduse from muse.quantities import maximum_production - from muse.utilities import agent_concatenation, broadcast_techs, reduce_assets + from muse.utilities import agent_concatenation, broadcast_over_assets, reduce_assets current_year, investment_year = map(int, market.year.values) @@ -264,7 +264,7 @@ def decommissioning(capacity, technologies): ) # Select technodata for assets - technodata = broadcast_techs(technologies, capacity, installed_as_year=True) + technodata = broadcast_over_assets(technologies, capacity, installed_as_year=True) demands = new_and_retro_demands( capacity, @@ -380,7 +380,7 @@ def standard_demand( from muse.commodities import is_enduse from muse.quantities import maximum_production - from muse.utilities import agent_concatenation, broadcast_techs, reduce_assets + from muse.utilities import agent_concatenation, broadcast_over_assets, reduce_assets current_year, investment_year = map(int, market.year.values) @@ -404,7 +404,7 @@ def decommissioning(capacity, technologies): ) # Select technodata for assets - technodata = broadcast_techs(technologies, capacity, installed_as_year=True) + technodata = broadcast_over_assets(technologies, capacity, installed_as_year=True) # Calculate new and retrofit demands demands = new_and_retro_demands( @@ -477,7 +477,7 @@ def unmet_forecasted_demand( ) -> xr.DataArray: """Forecast demand that cannot be serviced by non-decommissioned current assets.""" from muse.commodities import is_enduse - from muse.utilities import broadcast_techs, reduce_assets + from muse.utilities import broadcast_over_assets, reduce_assets current_year, investment_year = map(int, market.year.values) @@ -494,7 +494,7 @@ def unmet_forecasted_demand( future_capacity = capacity.sel(year=investment_year) # Select technology data for assets - techs = broadcast_techs(technologies, capacity, installed_as_year=True) + techs = broadcast_over_assets(technologies, capacity, installed_as_year=True) # Calculate unmet demand result = unmet_demand( diff --git a/src/muse/examples.py b/src/muse/examples.py index f5a9949cb..3645fca66 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -264,13 +264,13 @@ def matching_market(sector: str, model: str = "default") -> xr.Dataset: from muse.examples import sector as load_sector from muse.quantities import consumption, maximum_production from muse.sectors import Sector - from muse.utilities import agent_concatenation, broadcast_techs + from muse.utilities import agent_concatenation, broadcast_over_assets loaded_sector = cast(Sector, load_sector(sector, model)) assets = agent_concatenation({u.uuid: u.assets for u in list(loaded_sector.agents)}) market = xr.Dataset() - techs = broadcast_techs(loaded_sector.technologies, assets.capacity) + techs = broadcast_over_assets(loaded_sector.technologies, assets.capacity) production = maximum_production(techs, assets.capacity) market["supply"] = production.sum("asset") if "dst_region" in market.dims: diff --git a/src/muse/investments.py b/src/muse/investments.py index 055c6811a..750a2349c 100644 --- a/src/muse/investments.py +++ b/src/muse/investments.py @@ -225,14 +225,14 @@ def adhoc_match_demand( ) -> xr.DataArray: from muse.demand_matching import demand_matching from muse.quantities import capacity_in_use, maximum_production - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets assert "year" not in technologies.dims demand = next(c for c in constraints if c.name == "demand").b max_capacity = next(c for c in constraints if c.name == "max capacity expansion").b - techs = broadcast_techs(technologies, max_capacity) + techs = broadcast_over_assets(technologies, max_capacity) max_prod = maximum_production( techs, max_capacity, diff --git a/src/muse/quantities.py b/src/muse/quantities.py index 90670351f..f97c92b42 100644 --- a/src/muse/quantities.py +++ b/src/muse/quantities.py @@ -43,7 +43,7 @@ def supply( input commodities). """ from muse.commodities import CommodityUsage, check_usage, is_pollutant - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets assert "asset" not in demand.dims assert "asset" in capacity.dims @@ -74,9 +74,9 @@ def supply( else: # Multi-region models - demand = broadcast_techs(demand, maxprod, installed_as_year=False) + demand = broadcast_over_assets(demand, maxprod, installed_as_year=False) total_maxprod_by_region = maxprod.groupby("region").sum(dim="asset") - share_by_asset = maxprod / broadcast_techs( + share_by_asset = maxprod / broadcast_over_assets( total_maxprod_by_region, maxprod, installed_as_year=False ) demand_by_asset = (demand * share_by_asset).fillna(0) @@ -226,8 +226,7 @@ def maximum_production( capacity: Capacity of each technology of interest. In practice, the capacity can refer to asset capacity, the max capacity, or the capacity-in-use. technologies: xr.Dataset describing the features of the technologies of - interests. It should contain `fixed_outputs` and `utilization_factor`. It's - shape is matched to `capacity` using `muse.utilities.broadcast_techs`. + interests. It should contain `fixed_outputs` and `utilization_factor`. filters: keyword arguments are used to filter down the capacity and technologies. Filters not relevant to the quantities of interest, i.e. filters that are not a dimension of `capacity` or `technologies`, are @@ -270,8 +269,7 @@ def capacity_in_use( Arguments: production: Production from each technology of interest. technologies: xr.Dataset describing the features of the technologies of - interests. It should contain `fixed_outputs` and `utilization_factor`. It's - shape is matched to `capacity` using `muse.utilities.broadcast_techs`. + interests. It should contain `fixed_outputs` and `utilization_factor`. max_dim: reduces the given dimensions using `max`. Defaults to "commodity". If None, then no reduction is performed. filters: keyword arguments are used to filter down the capacity and @@ -284,7 +282,7 @@ def capacity_in_use( Capacity-in-use for each technology, whittled down by the filters. """ from muse.commodities import is_enduse - from muse.utilities import broadcast_techs, filter_input + from muse.utilities import broadcast_over_assets, filter_input prod = filter_input( production, **{k: v for k, v in filters.items() if k in production.dims} @@ -292,7 +290,7 @@ def capacity_in_use( techs = technologies[["fixed_outputs", "utilization_factor"]] assert isinstance(techs, xr.Dataset) - btechs = broadcast_techs(techs, prod, installed_as_year=True) + btechs = broadcast_over_assets(techs, prod, installed_as_year=True) ftechs = filter_input( btechs, **{k: v for k, v in filters.items() if k in technologies.dims} ) @@ -337,7 +335,6 @@ def minimum_production( refer to asset capacity, the max capacity, or the capacity-in-use. technologies: xr.Dataset describing the features of the technologies of interests. It should contain `fixed_outputs` and `minimum_service_factor`. - Its shape is matched to `capacity` using `muse.utilities.broadcast_techs`. timeslices: xr.DataArray of the timeslicing scheme. Production data will be returned in this format. filters: keyword arguments are used to filter down the capacity and diff --git a/src/muse/sectors/sector.py b/src/muse/sectors/sector.py index a1e9d2d28..6204fd5ba 100644 --- a/src/muse/sectors/sector.py +++ b/src/muse/sectors/sector.py @@ -291,14 +291,16 @@ def market_variables(self, market: xr.Dataset, technologies: xr.Dataset) -> Any: from muse.commodities import is_pollutant from muse.costs import levelized_cost_of_energy, supply_cost from muse.quantities import consumption - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets years = market.year.values capacity = self.capacity.interp(year=years, **self.interpolation) # Select technology data for each asset # Each asset uses the technology data from the year it was installed - technodata = broadcast_techs(technologies, capacity, installed_as_year=True) + technodata = broadcast_over_assets( + technologies, capacity, installed_as_year=True + ) # Calculate supply supply = self.supply_prod( diff --git a/src/muse/utilities.py b/src/muse/utilities.py index 40b93b0ea..d6f5ab005 100644 --- a/src/muse/utilities.py +++ b/src/muse/utilities.py @@ -179,7 +179,7 @@ def operation(x): return result.drop_vars("asset") -def broadcast_techs( +def broadcast_over_assets( technologies: xr.Dataset | xr.DataArray, template: xr.DataArray | xr.Dataset, installed_as_year: bool = True, @@ -241,9 +241,9 @@ def broadcast_techs( capacity of each asset, for example. We want to select the values from the technology array that correspond to each - asset in the template. To do this, we perform `broadcast_techs` on + asset in the template. To do this, we perform `broadcast_over_assets` on `technologies` using `assets` as a template: - >>> broadcast_techs(technologies, assets, installed_as_year=False) + >>> broadcast_over_assets(technologies, assets, installed_as_year=False) Size: 16B array([1, 5]) Coordinates: @@ -337,22 +337,20 @@ def filter_with_template( """Filters data to match template. If the `asset_dimension` is present in `template.dims`, then the call is - forwarded to `broadcast_techs`. Otherwise, the set of dimensions and indices + forwarded to `broadcast_over_assets`. Otherwise, the set of dimensions and indices in common between `template` and `data` are determined, and the resulting call is forwarded to `filter_input`. Arguments: data: Data to transform template: Data from which to figure coordinates and dimensions - asset_dimension: Name of the dimension which if present indicates the - format is that of an *asset* (see `broadcast_techs`) - kwargs: passed on to `broadcast_techs` or `filter_input` + kwargs: passed on to `broadcast_over_assets` or `filter_input` Returns: `data` transformed to match the form of `template` """ if "asset" in template.dims: - return broadcast_techs(data, template) + return broadcast_over_assets(data, template) match_indices = set(data.dims).intersection(template.dims) - set(kwargs) match = {d: template[d].isin(data[d]).values for d in match_indices if d != "year"} diff --git a/tests/conftest.py b/tests/conftest.py index 60f6d7aed..1d3ce7d59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -383,7 +383,7 @@ def _stock( from numpy.random import choice, rand from xarray import Dataset - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets n_assets = 10 @@ -396,7 +396,7 @@ def _stock( assets = Dataset(coords=asset_coords) # Create random capacity data - capacity_limits = broadcast_techs(technologies.total_capacity_limit, assets) + capacity_limits = broadcast_over_assets(technologies.total_capacity_limit, assets) factors = cumprod(rand(n_assets, len(coords["year"])) / 4 + 0.75, axis=1).clip( max=1 ) diff --git a/tests/test_constraints.py b/tests/test_constraints.py index c9ad0f13e..336e519b6 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -58,10 +58,10 @@ def capacity(assets): @fixture def market_demand(assets, technologies): from muse.quantities import maximum_production - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets return 0.8 * maximum_production( - broadcast_techs(technologies, assets), + broadcast_over_assets(technologies, assets), assets.capacity, ).sel(year=INVESTMENT_YEAR).groupby("technology").sum("asset").rename( technology="asset" diff --git a/tests/test_costs.py b/tests/test_costs.py index 49b5cee00..1683a889a 100644 --- a/tests/test_costs.py +++ b/tests/test_costs.py @@ -18,18 +18,18 @@ def _capacity(_technologies, demand_share): @fixture def _technologies(technologies, demand_share): """Technology parameters for each asset.""" - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - return broadcast_techs(technologies.sel(year=YEAR), demand_share) + return broadcast_over_assets(technologies.sel(year=YEAR), demand_share) @fixture def _prices(market, demand_share): """Prices relevant to each asset.""" - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets prices = market.prices.sel(year=YEAR) - return broadcast_techs(prices, demand_share, installed_as_year=False) + return broadcast_over_assets(prices, demand_share, installed_as_year=False) @fixture diff --git a/tests/test_demand_share.py b/tests/test_demand_share.py index 125708405..569038fd0 100644 --- a/tests/test_demand_share.py +++ b/tests/test_demand_share.py @@ -21,9 +21,9 @@ def _technologies(technologies, _capacity): @fixture def _market(_technologies, _capacity, timeslice): """A market which matches stocks exactly.""" - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) return _matching_market(_technologies, _capacity).transpose( "timeslice", "region", "commodity", "year" ) @@ -55,18 +55,18 @@ def test_fixtures(_capacity, _market, _technologies): def test_new_retro_split_zero_unmet(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) share = new_and_retro_demands(_capacity, _market, _technologies) assert (share == 0).all() def test_new_retro_split_zero_consumption_increase(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) _market.consumption.loc[{"year": INVESTMENT_YEAR}] = _market.consumption.sel( year=CURRENT_YEAR @@ -97,9 +97,9 @@ def test_new_retro_split_zero_consumption_increase(_capacity, _market, _technolo def test_new_retro_split_zero_new_unmet(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) _market.consumption.loc[{"year": INVESTMENT_YEAR}] = _market.supply.sel( year=CURRENT_YEAR, drop=True @@ -135,9 +135,9 @@ def test_new_retro_split_zero_new_unmet(_capacity, _market, _technologies): def test_new_retro_accounting_identity(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands from muse.quantities import maximum_production - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) share = new_and_retro_demands(_capacity, _market, _technologies) assert (share >= 0).all() @@ -166,7 +166,7 @@ def test_new_retro_accounting_identity(_capacity, _market, _technologies): def test_demand_split(_capacity, _market, _technologies): from muse.commodities import is_enduse from muse.demand_share import _inner_split as inner_split - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets def method(capacity, technologies): from muse.demand_share import decommissioning_demand @@ -180,7 +180,7 @@ def method(capacity, technologies): year=INVESTMENT_YEAR, region="USA", drop=True ).where(is_enduse(_technologies.comm_usage.sel(commodity=_market.commodity))) agents = dict(scully=_capacity, mulder=_capacity) - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) technodata = dict(scully=_technologies, mulder=_technologies) quantity = dict(scully=("scully", "USA", 0.3), mulder=("mulder", "USA", 0.7)) share = inner_split(agents, technodata, demand, method, quantity) @@ -203,7 +203,7 @@ def test_demand_split_zero_share(_capacity, _market, _technologies): """See issue SgiModel/StarMuse#688.""" from muse.commodities import is_enduse from muse.demand_share import _inner_split as inner_split - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets def method(capacity, technologies): from muse.demand_share import decommissioning_demand @@ -217,7 +217,7 @@ def method(capacity, technologies): year=INVESTMENT_YEAR, region="USA", drop=True ).where(is_enduse(_technologies.comm_usage.sel(commodity=_market.commodity))) agents = dict(scully=0.3 * _capacity, mulder=0.7 * _capacity) - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) technodata = dict(scully=_technologies, mulder=_technologies) quantity = dict(scully=("scully", "USA", 1), mulder=("mulder", "USA", 1)) share = inner_split(agents, technodata, demand, method, quantity) @@ -244,16 +244,16 @@ def test_new_retro_demand_share(_technologies, market, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import new_and_retro - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets asia_stock = stock.where(stock.region == "ASEAN", drop=True) usa_stock = stock.where(stock.region == "USA", drop=True) asia_market = _matching_market( - broadcast_techs(_technologies, asia_stock), asia_stock.capacity + broadcast_over_assets(_technologies, asia_stock), asia_stock.capacity ) usa_market = _matching_market( - broadcast_techs(_technologies, usa_stock), usa_stock.capacity + broadcast_over_assets(_technologies, usa_stock), usa_stock.capacity ) market = xr.concat((asia_market, usa_market), dim="region") market.consumption.loc[{"year": 2030}] *= 2 @@ -303,16 +303,16 @@ def test_standard_demand_share(_technologies, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import standard_demand from muse.errors import RetrofitAgentInStandardDemandShare - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets asia_stock = stock.where(stock.region == "ASEAN", drop=True) usa_stock = stock.where(stock.region == "USA", drop=True) asia_market = _matching_market( - broadcast_techs(_technologies, asia_stock), asia_stock.capacity + broadcast_over_assets(_technologies, asia_stock), asia_stock.capacity ) usa_market = _matching_market( - broadcast_techs(_technologies, usa_stock), usa_stock.capacity + broadcast_over_assets(_technologies, usa_stock), usa_stock.capacity ) market = xr.concat((asia_market, usa_market), dim="region") market.consumption.loc[{"year": 2030}] *= 2 @@ -359,16 +359,16 @@ def test_unmet_forecast_demand(_technologies, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import unmet_forecasted_demand - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets asia_stock = stock.where(stock.region == "ASEAN", drop=True) usa_stock = stock.where(stock.region == "USA", drop=True) asia_market = _matching_market( - broadcast_techs(_technologies, asia_stock), asia_stock.capacity + broadcast_over_assets(_technologies, asia_stock), asia_stock.capacity ) usa_market = _matching_market( - broadcast_techs(_technologies, usa_stock), usa_stock.capacity + broadcast_over_assets(_technologies, usa_stock), usa_stock.capacity ) market = xr.concat((asia_market, usa_market), dim="region") @@ -419,9 +419,9 @@ class Agent: def test_decommissioning_demand(_technologies, _capacity, timeslice): from muse.commodities import is_enduse from muse.demand_share import decommissioning_demand - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - _technologies = broadcast_techs(_technologies, _capacity) + _technologies = broadcast_over_assets(_technologies, _capacity) _capacity.loc[{"year": CURRENT_YEAR}] = current = 1.3 _capacity.loc[{"year": INVESTMENT_YEAR}] = forecast = 1.0 diff --git a/tests/test_objectives.py b/tests/test_objectives.py index 17e0b527d..06a42fef0 100644 --- a/tests/test_objectives.py +++ b/tests/test_objectives.py @@ -10,18 +10,18 @@ def _demand(demand_share): @fixture def _technologies(technologies, demand_share): - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets techs = technologies.sel(year=YEAR).rename(technology="replacement") - return broadcast_techs(techs, demand_share) + return broadcast_over_assets(techs, demand_share) @fixture def _prices(market, demand_share): - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets prices = market.prices.sel(year=YEAR) - return broadcast_techs(prices, demand_share, installed_as_year=False) + return broadcast_over_assets(prices, demand_share, installed_as_year=False) def test_fixtures(_technologies, _demand, _prices): diff --git a/tests/test_quantities.py b/tests/test_quantities.py index 304a0daf0..372491ec7 100644 --- a/tests/test_quantities.py +++ b/tests/test_quantities.py @@ -5,9 +5,9 @@ @fixture def technologies(technologies, capacity, timeslice): - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - return broadcast_techs(technologies, capacity) + return broadcast_over_assets(technologies, capacity) @fixture @@ -234,8 +234,8 @@ def test_supply_with_min_service(technologies, capacity, production, timeslice): def test_production_amplitude(production, technologies): from muse.quantities import production_amplitude - from muse.utilities import broadcast_techs + from muse.utilities import broadcast_over_assets - techs = broadcast_techs(technologies, production) + techs = broadcast_over_assets(technologies, production) result = production_amplitude(production, techs) assert set(result.dims) == set(production.dims) - {"commodity"} diff --git a/tests/test_utilities.py b/tests/test_utilities.py index cb557ae46..d7767fc02 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -40,22 +40,22 @@ def test_reduce_assets_with_zero_size(capacity: xr.DataArray): assert (actual == x).all() -def test_broadcast_techs(technologies, capacity): - from muse.utilities import broadcast_techs +def test_broadcast_over_assets(technologies, capacity): + from muse.utilities import broadcast_over_assets # Test with installed_as_year = True - result1 = broadcast_techs(technologies, capacity, installed_as_year=True) + result1 = broadcast_over_assets(technologies, capacity, installed_as_year=True) assert set(result1.dims) == {"asset", "commodity"} assert (result1.asset == capacity.asset).all() # Test with installed_as_year = False - result2 = broadcast_techs(technologies, capacity, installed_as_year=False) + result2 = broadcast_over_assets(technologies, capacity, installed_as_year=False) assert set(result2.dims) == {"asset", "commodity", "year"} assert (result2.asset == capacity.asset).all() # Template without "asset" dimensions (TODO: need to make the function stricter) # with raises(AssertionError): - # broadcast_techs(technologies, technologies) + # broadcast_over_assets(technologies, technologies) def test_tupled_dimension_no_tupling(): From 8792cb148446f5eb4637d17017f071c68dc6e09d Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 4 Feb 2025 10:46:47 +0000 Subject: [PATCH 10/14] Better docstring, rename argument --- src/muse/utilities.py | 68 ++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/src/muse/utilities.py b/src/muse/utilities.py index d6f5ab005..51003cfe1 100644 --- a/src/muse/utilities.py +++ b/src/muse/utilities.py @@ -180,41 +180,45 @@ def operation(x): def broadcast_over_assets( - technologies: xr.Dataset | xr.DataArray, + data: xr.Dataset | xr.DataArray, template: xr.DataArray | xr.Dataset, installed_as_year: bool = True, ) -> xr.Dataset | xr.DataArray: - """Broadcasts technologies to the shape of template in given dimension. + """Broadcasts an array to the shape of a template containing asset-level data. - The dimensions of the technologies are fully explicit, in that each concept - 'technology', 'region', 'year' (for year of issue) is a separate dimension. - However, the dataset or data arrays representing other quantities, such as - capacity, are often flattened out with coordinates 'region', 'installed', - and 'technology' represented in a single 'asset' dimension. This latter - representation is sparse if not all combinations of 'region', 'installed', - and 'technology' are present, whereas the former representation makes it - easier to select a subset of the same. + The dimensions of many arrays (such as technology datasets) are fully explicit, in + that each concept (e.g. 'technology', 'region', 'year') is a separate dimension. + However, other datasets (e.g capacity), are presented on a per-asset basis, + containing a single 'asset' dimension with with coordinates such as 'region', + 'installed' (year of installation), and 'technology'. This latter representation is + sparse if not all combinations of 'region', 'installed' and 'technology' are + present. - This function broadcast the first representation to the shape and coordinates - of the second. + This function broadcasts the first representation to the shape and coordinates + of the second, selecting the appropriate values for each asset (see example below). - Note: this is not necessarily limited to `technology` datasets. For + Note: this is not necessarily limited to technology datasets. For example, it could also be used on a dataset of commodity prices to select prices - relevant to each asset (e.g. if assets exist in multiple regions). In this example, - installed_as_year should be set to False (see below). + relevant to each asset (e.g. if assets exist in multiple regions). Arguments: - technologies: The dataset to broadcast - template: the dataset or data-array to use as a template - installed_as_year: True means that the "year" dimension in the technologies - dataset corresponds to the year that the asset was installed. Will commonly - be True for most technology parameters (e.g. var_par/fix_par are specified - the year that an asset is installed, and fixed for the lifetime of the - asset). If True, the technologies dataset must have data for every possible - "installed" year in the template. + data: The dataset/data-array to broadcast + template: The dataset/data-array to use as a template + installed_as_year: True means that the "year" dimension in 'data` + corresponds to the year that the asset was installed. This will commonly + be the case for most technology parameters (e.g. var_par/fix_par are + specified the year that an asset is installed, and fixed for the lifetime of + the asset). In this case, `data` must have a year coordinate for every + possible "installed" year in the template. + + Conversely, if the values in `data` apply to the year of activity, rather + than the year of installation, `installed_as_year` should be False. + An example would be commodity prices, which can change over the lifetime + of an asset. In this case, if "year" is present as a dimension in `data`, + it will be maintained as a separate dimension in the output. Example: - Define the technology array: + Define the data array: >>> import xarray as xr >>> technologies = xr.DataArray( ... data=[[1, 2, 3], [4, 5, 6]], @@ -255,7 +259,7 @@ def broadcast_over_assets( in the output is the value in the original technology array that matches the technology & region of each asset. """ - # TODO: this will return `technologies` unchanged if the template has no "asset" + # TODO: this will return `data` unchanged if the template has no "asset" # dimension, but strictly speaking we shouldn't allow this. # assert "asset" in template.dims @@ -267,18 +271,16 @@ def broadcast_over_assets( # TODO: this should be stricter, and enforce that the template has "installed" data, # and that the technologies dataset has a "year" dimension. # if installed_as_year: - if installed_as_year and "installed" in names and "year" in technologies.dims: + if installed_as_year and "installed" in names and "year" in data.dims: # assert "installed" in names - technologies = technologies.rename(year="installed") + data = data.rename(year="installed") - # The first selection reduces the size of technologies without affecting the + # The first selection reduces the size of the data without affecting the # dimensions. - first_sel = { - n: technologies[n].isin(template[n]) for n in names if n in technologies.dims - } - techs = technologies.sel(first_sel) + first_sel = {n: data[n].isin(template[n]) for n in names if n in data.dims} + techs = data.sel(first_sel) - # Reshape the technology array to match the template + # Reshape the array to match the template second_sel = {n: template[n] for n in template.coords if n in techs.dims} return techs.sel(second_sel) From afd28a0fd88936f012419020e1c3345b92f1aac7 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 4 Feb 2025 11:00:32 +0000 Subject: [PATCH 11/14] Delete filter_with_template --- src/muse/quantities.py | 15 ++++----------- src/muse/utilities.py | 30 ------------------------------ 2 files changed, 4 insertions(+), 41 deletions(-) diff --git a/src/muse/quantities.py b/src/muse/quantities.py index f97c92b42..7f7bb2538 100644 --- a/src/muse/quantities.py +++ b/src/muse/quantities.py @@ -158,24 +158,17 @@ def consumption( dimensions as `production`. """ - from muse.utilities import filter_with_template - - params = filter_with_template( - technologies[["fixed_inputs", "flexible_inputs", "fixed_outputs"]], - production, - ) - # Calculate degree of technology activity - prod_amplitude = production_amplitude(production, params) + prod_amplitude = production_amplitude(production, technologies) # Calculate consumption of fixed commodities consumption_fixed = prod_amplitude * broadcast_timeslice( - params.fixed_inputs, level=timeslice_level + technologies.fixed_inputs, level=timeslice_level ) assert all(consumption_fixed.commodity.values == production.commodity.values) # If there are no flexible inputs, then we are done - if not (params.flexible_inputs > 0).any(): + if not (technologies.flexible_inputs > 0).any(): return consumption_fixed # If prices are not given, then we can't consider flexible inputs, so just return @@ -184,7 +177,7 @@ def consumption( return consumption_fixed # Flexible inputs - flexs = broadcast_timeslice(params.flexible_inputs, level=timeslice_level) + flexs = broadcast_timeslice(technologies.flexible_inputs, level=timeslice_level) # Calculate the cheapest fuel for each flexible technology priceflex = prices * flexs diff --git a/src/muse/utilities.py b/src/muse/utilities.py index 51003cfe1..8f735e250 100644 --- a/src/muse/utilities.py +++ b/src/muse/utilities.py @@ -331,36 +331,6 @@ def filter_input( return dataset -def filter_with_template( - data: xr.Dataset | xr.DataArray, - template: xr.DataArray | xr.Dataset, - **kwargs, -): - """Filters data to match template. - - If the `asset_dimension` is present in `template.dims`, then the call is - forwarded to `broadcast_over_assets`. Otherwise, the set of dimensions and indices - in common between `template` and `data` are determined, and the resulting - call is forwarded to `filter_input`. - - Arguments: - data: Data to transform - template: Data from which to figure coordinates and dimensions - kwargs: passed on to `broadcast_over_assets` or `filter_input` - - Returns: - `data` transformed to match the form of `template` - """ - if "asset" in template.dims: - return broadcast_over_assets(data, template) - - match_indices = set(data.dims).intersection(template.dims) - set(kwargs) - match = {d: template[d].isin(data[d]).values for d in match_indices if d != "year"} - if "year" in match_indices: - match["year"] = template.year.values - return filter_input(data, **match, **kwargs) # type: ignore - - def tupled_dimension(array: np.ndarray, axis: int): """Transforms one axis into a tuples.""" if array.shape[axis] == 1: From 63479b1365095f86e19e85bfb3efdb722f410a01 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 4 Feb 2025 11:07:47 +0000 Subject: [PATCH 12/14] Remove `broadcast_over_assets` from `capacity_in_use` --- src/muse/investments.py | 5 ++--- src/muse/quantities.py | 7 ++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/muse/investments.py b/src/muse/investments.py index 750a2349c..eee255693 100644 --- a/src/muse/investments.py +++ b/src/muse/investments.py @@ -232,9 +232,8 @@ def adhoc_match_demand( demand = next(c for c in constraints if c.name == "demand").b max_capacity = next(c for c in constraints if c.name == "max capacity expansion").b - techs = broadcast_over_assets(technologies, max_capacity) max_prod = maximum_production( - techs, + broadcast_over_assets(technologies, max_capacity), max_capacity, technology=costs.replacement, commodity=demand.commodity, @@ -256,7 +255,7 @@ def adhoc_match_demand( capacity = capacity_in_use( production, - technologies, + broadcast_over_assets(technologies, production), technology=production.replacement, timeslice_level=timeslice_level, ).drop_vars("technology") diff --git a/src/muse/quantities.py b/src/muse/quantities.py index 7f7bb2538..147e8f94f 100644 --- a/src/muse/quantities.py +++ b/src/muse/quantities.py @@ -275,17 +275,14 @@ def capacity_in_use( Capacity-in-use for each technology, whittled down by the filters. """ from muse.commodities import is_enduse - from muse.utilities import broadcast_over_assets, filter_input + from muse.utilities import filter_input prod = filter_input( production, **{k: v for k, v in filters.items() if k in production.dims} ) - techs = technologies[["fixed_outputs", "utilization_factor"]] - assert isinstance(techs, xr.Dataset) - btechs = broadcast_over_assets(techs, prod, installed_as_year=True) ftechs = filter_input( - btechs, **{k: v for k, v in filters.items() if k in technologies.dims} + technologies, **{k: v for k, v in filters.items() if k in technologies.dims} ) factor = 1 / (ftechs.fixed_outputs * ftechs.utilization_factor) From b7b1b8987a644ea8e4cb4fc8429ba5190e17acdc Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 4 Feb 2025 11:22:28 +0000 Subject: [PATCH 13/14] Fix trade tests --- src/muse/examples.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/muse/examples.py b/src/muse/examples.py index 3645fca66..3aa3e2878 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -276,7 +276,7 @@ def matching_market(sector: str, model: str = "default") -> xr.Dataset: if "dst_region" in market.dims: market = market.rename(dst_region="region") if market.region.dims: - consump = consumption(loaded_sector.technologies, production) + consump = consumption(techs, production) market["consumption"] = drop_timeslice( consump.groupby("region").sum( {"asset", "dst_region"}.intersection(consump.dims) @@ -285,7 +285,7 @@ def matching_market(sector: str, model: str = "default") -> xr.Dataset: ) else: market["consumption"] = ( - consumption(loaded_sector.technologies, production).sum( + consumption(techs, production).sum( {"asset", "dst_region"}.intersection(market.dims) ) + market.supply From a8a893487e903b200f915ec8f818a61059f32d30 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 4 Feb 2025 11:43:11 +0000 Subject: [PATCH 14/14] Edit TODO --- tests/test_quantities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_quantities.py b/tests/test_quantities.py index 372491ec7..1877f47e1 100644 --- a/tests/test_quantities.py +++ b/tests/test_quantities.py @@ -94,8 +94,8 @@ def test_production_agent_asset_view( ): """Production when capacity has format of agent.assets.capacity. - TODO: not currently supported. Need to make maximum_production more generic so it - can handle capacity data without an "asset" dimension. + TODO: This requires a fully-explicit technologies dataset. Need to rework the + fixtures. """ from muse.utilities import coords_to_multiindex, reduce_assets