From 9e1b60667e6c3e6635fd7471a69f1f2c9309993b Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 28 Jan 2025 10:45:46 +0000 Subject: [PATCH 01/12] Check array dimensions after multiplication --- src/muse/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/muse/__main__.py b/src/muse/__main__.py index 21c86a8f7..e5854ba91 100644 --- a/src/muse/__main__.py +++ b/src/muse/__main__.py @@ -102,6 +102,13 @@ def patched_broadcast_compat_data(self, other): self_data = self.data other_data = other dims = self.dims + + # Check output dimensions + if "asset" in dims and any( + dim in dims for dim in ["region", "technology", "installed"] + ): + raise ValueError() + return self_data, other_data, dims From 4394071dd42d6c5bdda03bce07f553d9a1f07aaa Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 28 Jan 2025 11:50:53 +0000 Subject: [PATCH 02/12] Fix tests in costs module --- tests/conftest.py | 19 +++++++++--- tests/test_costs.py | 70 ++++++++++++++++++++++++--------------------- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index bef2e1442..68890f1c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -446,15 +446,26 @@ def search_space(retro_agent, technologies): @fixture def demand_share(coords, timeslice): """Example demand share, as would be computed by an agent.""" - from numpy.random import rand + from numpy.random import choice, rand + n_assets = 5 axes = { "commodity": coords["commodity"], - "asset": list(set(coords["technology"])), "timeslice": timeslice.timeslice, + "technology": (["asset"], choice(coords["technology"], n_assets, replace=True)), + "region": (["asset"], choice(coords["region"], n_assets, replace=True)), } - shape = len(axes["commodity"]), len(axes["asset"]), len(axes["timeslice"]) - result = DataArray(rand(*shape), coords=axes, dims=axes.keys(), name="demand_share") + shape = ( + len(axes["commodity"]), + len(axes["timeslice"]), + n_assets, + ) + result = DataArray( + rand(*shape), + dims=["commodity", "timeslice", "asset"], + coords=axes, + name="demand_share", + ) return result diff --git a/tests/test_costs.py b/tests/test_costs.py index 2e2cb86ee..fc49fd2b9 100644 --- a/tests/test_costs.py +++ b/tests/test_costs.py @@ -5,28 +5,27 @@ @fixture -def _prices(market): - prices = market.prices - return prices.sel(year=YEAR) - +def _capacity(technologies, demand_share): + """Capacity for a set of assets.""" + from muse.quantities import capacity_to_service_demand + from muse.utilities import broadcast_techs -@fixture -def _technologies(technologies): - return technologies.sel(year=YEAR) + techs = broadcast_techs(technologies.sel(year=YEAR), demand_share) + capacity = capacity_to_service_demand(technologies=techs, demand=demand_share) + return capacity @fixture -def _capacity(_technologies, demand_share): - from muse.quantities import capacity_to_service_demand +def _technologies(technologies, _capacity): + """Technology parameters for each asset.""" + from muse.utilities import broadcast_techs - capacity = capacity_to_service_demand( - technologies=_technologies, demand=demand_share - ) - return capacity + return broadcast_techs(technologies.sel(year=YEAR), _capacity) @fixture def _production(_technologies, _capacity): + """Production data for each asset.""" from muse.timeslices import broadcast_timeslice, distribute_timeslice production = ( @@ -39,6 +38,7 @@ def _production(_technologies, _capacity): @fixture def _consumption(_technologies, _capacity): + """Consumption data for each asset.""" from muse.timeslices import broadcast_timeslice, distribute_timeslice consumption = ( @@ -49,19 +49,26 @@ def _consumption(_technologies, _capacity): return consumption +@fixture +def _prices(market, _capacity): + """Prices relevant to each asset.""" + from muse.utilities import broadcast_techs + + prices = market.prices.sel(year=YEAR) + return broadcast_techs(prices, _capacity) + + def test_fixtures(_technologies, _prices, _capacity, _production, _consumption): """Validating that the fixtures have appropriate dimensions.""" - assert set(_technologies.dims) == {"commodity", "region", "technology"} - assert set(_prices.dims) == {"commodity", "region", "timeslice"} - assert set(_capacity.dims) == {"asset", "region", "technology"} + assert set(_technologies.dims) == {"asset", "commodity"} + assert set(_prices.dims) == {"asset", "commodity", "timeslice"} + assert set(_capacity.dims) == {"asset"} assert ( set(_production.dims) == set(_consumption.dims) == { "asset", "commodity", - "region", - "technology", "timeslice", } ) @@ -71,49 +78,49 @@ def test_capital_costs(_technologies, _capacity): from muse.costs import capital_costs result = capital_costs(_technologies, _capacity) - assert set(result.dims) == {"asset", "region", "technology"} + assert set(result.dims) == {"asset"} def test_environmental_costs(_technologies, _prices, _production): from muse.costs import environmental_costs result = environmental_costs(_technologies, _prices, _production) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} def test_fuel_costs(_technologies, _prices, _consumption): from muse.costs import fuel_costs result = fuel_costs(_technologies, _prices, _consumption) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} def test_material_costs(_technologies, _prices, _consumption): from muse.costs import material_costs result = material_costs(_technologies, _prices, _consumption) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} def test_fixed_costs(_technologies, _capacity): from muse.costs import fixed_costs result = fixed_costs(_technologies, _capacity) - assert set(result.dims) == {"asset", "region", "technology"} + assert set(result.dims) == {"asset"} def test_variable_costs(_technologies, _production): from muse.costs import variable_costs result = variable_costs(_technologies, _production) - assert set(result.dims) == {"asset", "region", "technology"} + assert set(result.dims) == {"asset"} def test_running_costs(_technologies, _prices, _capacity, _production, _consumption): from muse.costs import running_costs result = running_costs(_technologies, _prices, _capacity, _production, _consumption) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} def test_net_present_value( @@ -124,7 +131,7 @@ def test_net_present_value( result = net_present_value( _technologies, _prices, _capacity, _production, _consumption ) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} def test_net_present_cost(_technologies, _prices, _capacity, _production, _consumption): @@ -133,7 +140,7 @@ def test_net_present_cost(_technologies, _prices, _capacity, _production, _consu result = net_present_cost( _technologies, _prices, _capacity, _production, _consumption ) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} def test_equivalent_annual_cost( @@ -144,7 +151,7 @@ def test_equivalent_annual_cost( result = equivalent_annual_cost( _technologies, _prices, _capacity, _production, _consumption ) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} @mark.parametrize("method", ["annual", "lifetime"]) @@ -156,7 +163,7 @@ def test_levelized_cost_of_energy( result = levelized_cost_of_energy( _technologies, _prices, _capacity, _production, _consumption, method=method ) - assert set(result.dims) == {"asset", "region", "technology", "timeslice"} + assert set(result.dims) == {"asset", "timeslice"} def test_supply_cost(_technologies, _prices, _capacity, _production, _consumption): @@ -169,7 +176,6 @@ def test_supply_cost(_technologies, _prices, _capacity, _production, _consumptio assert set(result.dims) == { "commodity", "region", - "technology", "timeslice", } @@ -359,7 +365,7 @@ def test_lcoe_aggregate( method=method, aggregate_timeslices=True, ) - assert set(result.dims) == {"asset", "region", "technology"} # no timeslice dim + assert set(result.dims) == {"asset"} # no timeslice dim def test_npv_aggregate(_technologies, _prices, _capacity, _production, _consumption): @@ -373,4 +379,4 @@ def test_npv_aggregate(_technologies, _prices, _capacity, _production, _consumpt _consumption, aggregate_timeslices=True, ) - assert set(result.dims) == {"asset", "region", "technology"} # no timeslice dim + assert set(result.dims) == {"asset"} # no timeslice dim From c1f97c01f716bede51683dbd7422a7f35e189e47 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 28 Jan 2025 13:20:42 +0000 Subject: [PATCH 03/12] Fix objectives tests --- src/muse/objectives.py | 2 +- tests/conftest.py | 9 ++------- tests/test_objectives.py | 34 ++++++++++++++-------------------- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/src/muse/objectives.py b/src/muse/objectives.py index 0f5cabc7b..1d521c1ce 100644 --- a/src/muse/objectives.py +++ b/src/muse/objectives.py @@ -179,7 +179,7 @@ def decorated_objective( demand, ["asset", "timeslice", "commodity"], optional=["region"] ) check_dimensions( - technologies, ["replacement", "commodity"], optional=["timeslice"] + technologies, ["replacement", "commodity"], optional=["timeslice", "asset"] ) # Calculate objective diff --git a/tests/conftest.py b/tests/conftest.py index 68890f1c5..28544b32d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -290,7 +290,7 @@ def var(*dims, factor=100.0): @fixture -def agent_market(coords, technologies, timeslice) -> Dataset: +def agent_market(coords, timeslice) -> Dataset: from numpy.random import rand result = Dataset(coords=timeslice.coords) @@ -312,7 +312,7 @@ def var(*dims, factor=100.0): @fixture -def market(coords, technologies, timeslice) -> Dataset: +def market(coords, timeslice) -> Dataset: from numpy.random import rand result = Dataset(coords=timeslice.coords) @@ -371,11 +371,6 @@ def stock(coords, technologies) -> Dataset: return _stock(coords, technologies) -@fixture -def stock_factory() -> Callable: - return _stock - - def _stock( coords, technologies, diff --git a/tests/test_objectives.py b/tests/test_objectives.py index 9f56aadc5..8e0a71849 100644 --- a/tests/test_objectives.py +++ b/tests/test_objectives.py @@ -4,37 +4,31 @@ @fixture -def _technologies(technologies, retro_agent, search_space): - techs = retro_agent.filter_input( - technologies, - technology=search_space.replacement, - ).drop_vars("technology") - return techs.sel(year=YEAR) +def _demand(demand_share): + return demand_share @fixture -def _demand(demand_share, search_space): - reduced_demand = demand_share.sel( - { - k: search_space[k] - for k in set(demand_share.dims).intersection(search_space.dims) - } - ) - reduced_demand["year"] = 2030 - return reduced_demand +def _technologies(technologies, demand_share): + from muse.utilities import broadcast_techs + + techs = technologies.sel(year=YEAR).rename(technology="replacement") + return broadcast_techs(techs, demand_share) @fixture -def _prices(retro_agent, agent_market): - prices = retro_agent.filter_input(agent_market.prices) - return prices.sel(year=YEAR) +def _prices(market, demand_share): + from muse.utilities import broadcast_techs + + prices = market.prices.sel(year=YEAR) + return broadcast_techs(prices, demand_share) def test_fixtures(_technologies, _demand, _prices): """Validating that the fixtures have appropriate dimensions.""" - assert set(_technologies.dims) == {"commodity", "replacement"} + assert set(_technologies.dims) == {"asset", "commodity", "replacement"} assert set(_demand.dims) == {"asset", "commodity", "timeslice"} - assert set(_prices.dims) == {"commodity", "timeslice"} + assert set(_prices.dims) == {"asset", "commodity", "timeslice"} @mark.usefixtures("save_registries") From 400ebe0394fa79a7f9af5d39fdd79220f6eb02ce Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 28 Jan 2025 17:10:52 +0000 Subject: [PATCH 04/12] Fix demand_share tests --- tests/conftest.py | 73 +++++++++++----------------------- tests/test_costs.py | 32 +++++++-------- tests/test_demand_share.py | 80 ++++++++++++++++++++++---------------- 3 files changed, 84 insertions(+), 101 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 28544b32d..ed793215c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from pathlib import Path -from typing import Callable, Optional +from typing import Callable from unittest.mock import patch import numpy as np @@ -374,70 +374,41 @@ def stock(coords, technologies) -> Dataset: def _stock( coords, technologies, - region: Optional[Sequence[str]] = None, - nassets: Optional[int] = None, ) -> Dataset: from numpy import cumprod, stack - from numpy.random import choice, rand, randint + from numpy.random import choice, rand from xarray import Dataset + from muse.utilities import broadcast_techs + + n_assets = 10 ymin, ymax = min(coords["year"]), max(coords["year"]) + 1 - if nassets is None: - nmin = max(1, 0 if region is None else len(region)) - n = randint(nmin, max(nmin, 10)) - else: - n = nassets - tech_subset = choice( - coords["technology"], randint(2, len(coords["technology"])), replace=False - ) - asset = { - (choice(tech_subset), choice(range(ymin, min(ymin + 3, ymax)))) - for u in range(2 * n) + # Create assets + asset_coords = { + "technology": ("asset", choice(coords["technology"], n_assets, replace=True)), + "region": ("asset", choice(coords["region"], n_assets, replace=True)), + "installed": ("asset", choice(range(ymin, ymax), n_assets)), } - technology = [u[0] for u in asset][:n] - installed = [u[1] for u in asset][:n] - - factors = cumprod( - rand(len(installed), len(coords["year"])) / 4 + 0.75, axis=1 - ).clip(max=1) - capacity = 0.75 * technologies.total_capacity_limit.sel( - technology=technology, year=2010, region="USA", drop=True + assets = Dataset(coords=asset_coords) + + # Create random capacity data + capacity_limits = broadcast_techs(technologies.total_capacity_limit, assets) + factors = cumprod(rand(n_assets, len(coords["year"])) / 4 + 0.75, axis=1).clip( + max=1 ) capacity = stack( - [capacity * factors[:, i] for i in range(factors.shape[1])], axis=1 + [0.75 * capacity_limits * factors[:, i] for i in range(factors.shape[1])], + axis=1, ) - result = Dataset() - result["technology"] = "asset", technology - result["installed"] = "asset", installed - if region is not None and len(region) > 0: - result["region"] = "asset", choice(region, len(installed)) - result["year"] = "year", [ymin, max(max(installed), ymax)] + # Create capacity dataset + result = assets.copy() + result["year"] = "year", [ymin, ymax] result["capacity"] = ("asset", "year"), capacity - result = result.set_coords(("technology", "installed")) - if "region" in result.data_vars: - result = result.set_coords("region") return result -@fixture -def search_space(retro_agent, technologies): - """Example search space, as would be computed by an agent.""" - from numpy.random import randint - - coords = { - "asset": list(set(retro_agent.assets.technology.values)), - "replacement": technologies.technology.values, - } - return DataArray( - randint(0, 4, tuple(len(u) for u in coords.values())) == 0, - coords=coords, - dims=coords.keys(), - name="search_space", - ) - - @fixture def demand_share(coords, timeslice): """Example demand share, as would be computed by an agent.""" diff --git a/tests/test_costs.py b/tests/test_costs.py index fc49fd2b9..1769cd286 100644 --- a/tests/test_costs.py +++ b/tests/test_costs.py @@ -5,22 +5,31 @@ @fixture -def _capacity(technologies, demand_share): - """Capacity for a set of assets.""" +def _capacity(_technologies, demand_share): + """Capacity for each asset.""" from muse.quantities import capacity_to_service_demand - from muse.utilities import broadcast_techs - techs = broadcast_techs(technologies.sel(year=YEAR), demand_share) - capacity = capacity_to_service_demand(technologies=techs, demand=demand_share) + capacity = capacity_to_service_demand( + technologies=_technologies, demand=demand_share + ) return capacity @fixture -def _technologies(technologies, _capacity): +def _technologies(technologies, demand_share): """Technology parameters for each asset.""" from muse.utilities import broadcast_techs - return broadcast_techs(technologies.sel(year=YEAR), _capacity) + return broadcast_techs(technologies.sel(year=YEAR), demand_share) + + +@fixture +def _prices(market, demand_share): + """Prices relevant to each asset.""" + from muse.utilities import broadcast_techs + + prices = market.prices.sel(year=YEAR) + return broadcast_techs(prices, demand_share) @fixture @@ -49,15 +58,6 @@ def _consumption(_technologies, _capacity): return consumption -@fixture -def _prices(market, _capacity): - """Prices relevant to each asset.""" - from muse.utilities import broadcast_techs - - prices = market.prices.sel(year=YEAR) - return broadcast_techs(prices, _capacity) - - def test_fixtures(_technologies, _prices, _capacity, _production, _consumption): """Validating that the fixtures have appropriate dimensions.""" assert set(_technologies.dims) == {"asset", "commodity"} diff --git a/tests/test_demand_share.py b/tests/test_demand_share.py index 859a3abe6..92ca11981 100644 --- a/tests/test_demand_share.py +++ b/tests/test_demand_share.py @@ -8,18 +8,22 @@ @fixture -def _technologies(technologies): - return technologies.interp(year=INVESTMENT_YEAR) +def _capacity(stock): + return stock.capacity.interp(year=[CURRENT_YEAR, INVESTMENT_YEAR]) @fixture -def _capacity(stock): - return stock.capacity.interp(year=[CURRENT_YEAR, INVESTMENT_YEAR]) +def _technologies(technologies, _capacity): + """Technology parameters for the sector.""" + return technologies.interp(year=INVESTMENT_YEAR) @fixture def _market(_technologies, _capacity, timeslice): """A market which matches stocks exactly.""" + from muse.utilities import broadcast_techs + + _technologies = broadcast_techs(_technologies, _capacity) return _matching_market(_technologies, _capacity).transpose( "timeslice", "region", "commodity", "year" ) @@ -33,14 +37,22 @@ def _matching_market(technologies, capacity): market = xr.Dataset() production = maximum_production(technologies, capacity) + consumption = consumption(technologies, production) + if "region" in production.coords: + production = production.groupby("region") + consumption = consumption.groupby("region") market["supply"] = production.sum("asset") - market["consumption"] = drop_timeslice( - consumption(technologies, production).sum("asset") + market.supply - ) + market["consumption"] = drop_timeslice(consumption.sum("asset") + market.supply) market["prices"] = market.supply.dims, random(market.supply.shape) return market +def test_fixtures(_capacity, _market, _technologies): + assert set(_capacity.dims) == {"asset", "year"} + assert set(_market.dims) == {"commodity", "region", "year", "timeslice"} + assert set(_technologies.dims) == {"technology", "region", "commodity"} + + def test_new_retro_split_zero_unmet(_capacity, _market, _technologies): from muse.demand_share import new_and_retro_demands @@ -216,8 +228,8 @@ def test_new_retro_demand_share(_technologies, market, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import new_and_retro - asia_stock = stock.expand_dims(region=["ASEAN"]) - usa_stock = stock.expand_dims(region=["USA"]) + asia_stock = stock.where(stock.region == "ASEAN", drop=True) + usa_stock = stock.where(stock.region == "USA", drop=True) asia_market = _matching_market(_technologies, asia_stock.capacity) usa_market = _matching_market(_technologies, usa_stock.capacity) @@ -235,12 +247,12 @@ class Agent: quantity: float agents = [ - Agent(0.3 * usa_stock.squeeze("region"), "retrofit", uuid4(), "a", "USA", 0.3), - Agent(0.0 * usa_stock.squeeze("region"), "new", uuid4(), "a", "USA", 0.0), - Agent(0.7 * usa_stock.squeeze("region"), "retrofit", uuid4(), "b", "USA", 0.7), - Agent(0.0 * usa_stock.squeeze("region"), "new", uuid4(), "b", "USA", 0.0), - Agent(asia_stock.squeeze("region"), "retrofit", uuid4(), "a", "ASEAN", 1.0), - Agent(0 * asia_stock.squeeze("region"), "new", uuid4(), "a", "ASEAN", 0.0), + Agent(0.3 * usa_stock, "retrofit", uuid4(), "a", "USA", 0.3), + Agent(0.0 * usa_stock, "new", uuid4(), "a", "USA", 0.0), + Agent(0.7 * usa_stock, "retrofit", uuid4(), "b", "USA", 0.7), + Agent(0.0 * usa_stock, "new", uuid4(), "b", "USA", 0.0), + Agent(asia_stock, "retrofit", uuid4(), "a", "ASEAN", 1.0), + Agent(0 * asia_stock, "new", uuid4(), "a", "ASEAN", 0.0), ] results = new_and_retro(agents, market, _technologies) @@ -270,8 +282,8 @@ def test_standard_demand_share(_technologies, timeslice, stock): from muse.demand_share import standard_demand from muse.errors import RetrofitAgentInStandardDemandShare - asia_stock = stock.expand_dims(region=["ASEAN"]) - usa_stock = stock.expand_dims(region=["USA"]) + asia_stock = stock.where(stock.region == "ASEAN", drop=True) + usa_stock = stock.where(stock.region == "USA", drop=True) asia_market = _matching_market(_technologies, asia_stock.capacity) usa_market = _matching_market(_technologies, usa_stock.capacity) @@ -289,12 +301,12 @@ class Agent: quantity: float agents = [ - Agent(0.3 * usa_stock.squeeze("region"), "retrofit", uuid4(), "a", "USA", 0.3), - Agent(0.0 * usa_stock.squeeze("region"), "new", uuid4(), "a", "USA", 0.0), - Agent(0.7 * usa_stock.squeeze("region"), "retrofit", uuid4(), "b", "USA", 0.7), - Agent(0.0 * usa_stock.squeeze("region"), "new", uuid4(), "b", "USA", 0.0), - Agent(asia_stock.squeeze("region"), "retrofit", uuid4(), "a", "ASEAN", 1.0), - Agent(0 * asia_stock.squeeze("region"), "new", uuid4(), "a", "ASEAN", 0.0), + Agent(0.3 * usa_stock, "retrofit", uuid4(), "a", "USA", 0.3), + Agent(0.0 * usa_stock, "new", uuid4(), "a", "USA", 0.0), + Agent(0.7 * usa_stock, "retrofit", uuid4(), "b", "USA", 0.7), + Agent(0.0 * usa_stock, "new", uuid4(), "b", "USA", 0.0), + Agent(asia_stock, "retrofit", uuid4(), "a", "ASEAN", 1.0), + Agent(0 * asia_stock, "new", uuid4(), "a", "ASEAN", 0.0), ] with raises(RetrofitAgentInStandardDemandShare): @@ -321,8 +333,8 @@ def test_unmet_forecast_demand(_technologies, timeslice, stock): from muse.commodities import is_enduse from muse.demand_share import unmet_forecasted_demand - asia_stock = stock.expand_dims(region=["ASEAN"]) - usa_stock = stock.expand_dims(region=["USA"]) + asia_stock = stock.where(stock.region == "ASEAN", drop=True) + usa_stock = stock.where(stock.region == "USA", drop=True) asia_market = _matching_market(_technologies, asia_stock.capacity) usa_market = _matching_market(_technologies, usa_stock.capacity) @@ -335,9 +347,9 @@ class Agent: # First ensure that the demand is fully met agents = [ - Agent(0.3 * usa_stock.squeeze("region")), - Agent(0.7 * usa_stock.squeeze("region")), - Agent(asia_stock.squeeze("region")), + Agent(0.3 * usa_stock), + Agent(0.7 * usa_stock), + Agent(asia_stock), ] result = unmet_forecasted_demand(agents, market, _technologies) assert set(result.dims) == set(market.consumption.dims) - {"year"} @@ -345,9 +357,9 @@ class Agent: # Then try with too little demand agents = [ - Agent(0.4 * usa_stock.squeeze("region")), - Agent(0.8 * usa_stock.squeeze("region")), - Agent(1.1 * asia_stock.squeeze("region")), + Agent(0.4 * usa_stock), + Agent(0.8 * usa_stock), + Agent(1.1 * asia_stock), ] result = unmet_forecasted_demand( agents, @@ -359,8 +371,8 @@ class Agent: # Then try too little capacity agents = [ - Agent(0.5 * usa_stock.squeeze("region")), - Agent(0.5 * asia_stock.squeeze("region")), + Agent(0.5 * usa_stock), + Agent(0.5 * asia_stock), ] result = unmet_forecasted_demand(agents, market, _technologies) comm_usage = _technologies.comm_usage.sel(commodity=market.commodity) @@ -381,7 +393,7 @@ def test_decommissioning_demand(_technologies, _capacity, timeslice): _technologies.fixed_outputs[:] = fouts = 0.5 _technologies.utilization_factor[:] = ufac = 0.4 decom = decommissioning_demand(_technologies, _capacity) - assert set(decom.dims) == {"asset", "commodity", "region", "timeslice"} + assert set(decom.dims) == {"asset", "commodity", "timeslice"} assert decom.sel(commodity=is_enduse(_technologies.comm_usage)).sum( "timeslice" ).values == approx(ufac * fouts * (current - forecast)) From fdaab201d8901e4c4b9d861d36ae77d8047de909 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 29 Jan 2025 09:20:39 +0000 Subject: [PATCH 05/12] Fix quantities tests, delete gross_margin --- src/muse/quantities.py | 82 ---------------------------------------- tests/test_quantities.py | 51 ++++--------------------- 2 files changed, 7 insertions(+), 126 deletions(-) diff --git a/src/muse/quantities.py b/src/muse/quantities.py index 94c77ec32..8eca0dc2c 100644 --- a/src/muse/quantities.py +++ b/src/muse/quantities.py @@ -153,88 +153,6 @@ def emission( ) -def gross_margin( - technologies: xr.Dataset, - capacity: xr.DataArray, - prices: xr.Dataset, - timeslice_level: str | None = None, -) -> xr.DataArray: - """The percentage of revenue after direct expenses have been subtracted. - - .. _reference: - https://www.investopedia.com/terms/g/grossmargin.asp - We first calculate the revenues, which depend on prices - We then deduct the direct expenses - - energy commodities INPUTS are related to fuel costs - - environmental commodities OUTPUTS are related to environmental costs - - variable costs is given as technodata inputs - - non-environmental commodities OUTPUTS are related to revenues. - """ - from muse.commodities import is_enduse, is_pollutant - from muse.utilities import broadcast_techs - - tech = broadcast_techs( # type: ignore - cast( - xr.Dataset, - technologies[ - [ - "technical_life", - "interest_rate", - "var_par", - "var_exp", - "fixed_outputs", - "fixed_inputs", - ] - ], - ), - capacity, - ) - - var_par = tech.var_par - var_exp = tech.var_exp - fixed_outputs = tech.fixed_outputs - fixed_inputs = tech.fixed_inputs - # We separate the case where we have one or more regions - caparegions = np.array(capacity.region.values).reshape(-1) - if len(caparegions) > 1: - prices.sel(region=capacity.region) - else: - prices = prices.where(prices.region == capacity.region, drop=True) - prices = prices.interp(year=capacity.year.values) - - # Filters for pollutants and output commodities - environmentals = is_pollutant(technologies.comm_usage) - enduses = is_enduse(technologies.comm_usage) - - # Variable costs depend on factors such as labour - variable_costs = distribute_timeslice( - var_par * ((fixed_outputs.sel(commodity=enduses)).sum("commodity")) ** var_exp, - level=timeslice_level, - ) - - # The individual prices are selected - # costs due to consumables, direct inputs - consumption_costs = ( - prices * distribute_timeslice(fixed_inputs, level=timeslice_level) - ).sum("commodity") - # costs due to pollutants - production_costs = prices * distribute_timeslice( - fixed_outputs, level=timeslice_level - ) - environmental_costs = (production_costs.sel(commodity=environmentals)).sum( - "commodity" - ) - # revenues due to product sales - revenues = (production_costs.sel(commodity=enduses)).sum("commodity") - - # Gross margin is the net between revenues and all costs - result = revenues - environmental_costs - variable_costs - consumption_costs - - # Gross margin is defined as a ratio on revenues and as a percentage - result *= 100 / revenues - return result - - def consumption( technologies: xr.Dataset, production: xr.DataArray, diff --git a/tests/test_quantities.py b/tests/test_quantities.py index bdf75362b..2659f17ea 100644 --- a/tests/test_quantities.py +++ b/tests/test_quantities.py @@ -10,11 +10,13 @@ def production( technologies: xr.Dataset, capacity: xr.DataArray, timeslice ) -> xr.DataArray: from muse.timeslices import broadcast_timeslice, distribute_timeslice + from muse.utilities import broadcast_techs + techs = broadcast_techs(technologies, capacity) return ( broadcast_timeslice(capacity) - * distribute_timeslice(technologies.fixed_outputs) - * broadcast_timeslice(technologies.utilization_factor) + * distribute_timeslice(techs.fixed_outputs) + * broadcast_timeslice(techs.utilization_factor) ) @@ -55,47 +57,6 @@ def test_supply_emissions(technologies, capacity, timeslice): assert actual.values == approx(expected.values) -def test_gross_margin(technologies, capacity, market, timeslice): - from muse.commodities import is_enduse, is_fuel, is_pollutant - from muse.quantities import gross_margin - - """ - Gross margin refers to the calculation - .. _here: - https://www.investopedia.com/terms/g/grossmargin.asp - """ - # we modify the variables to have just the values we want for the testing - selected = capacity.technology.values[0] - - technologies = technologies.sel(technology=technologies.technology == selected) - capa = capacity.where(capacity.technology == selected, drop=True) - - # Filtering commodity outputs - usage = technologies.comm_usage - - technologies.var_par[:] = vp = 2 - technologies.var_exp[:] = ve = 0.5 - technologies.fixed_inputs[{"commodity": is_fuel(usage)}] = fuels = 2 - technologies.fixed_outputs[{"commodity": is_pollutant(usage)}] = envs = 10 - technologies.fixed_outputs[{"commodity": is_enduse(usage)}] = prod = 5 - - market.prices[:] = prices = 3 - market.prices[{"commodity": is_pollutant(usage)}] = env_prices = 6 - # We expect a xr.DataArray with 1 replacement technology - actual = gross_margin(technologies, capa, market.prices) - - revenues = prices * prod * sum(is_enduse(usage)) - env_costs = env_prices * envs * sum(is_pollutant(usage)) - cons_costs = prices * fuels * sum(is_fuel(usage)) - var_costs = vp * ((prod * sum(is_enduse(usage))) ** ve) - - expected = revenues - env_costs - cons_costs - var_costs - expected *= 100 / revenues - - expected, actual = xr.broadcast(expected, actual) - assert actual.values == approx(expected.values) - - def test_consumption(technologies, production, market): from muse.quantities import consumption @@ -273,6 +234,8 @@ def test_supply_capped_by_min_service(technologies, capacity, timeslice): def test_production_amplitude(production, technologies): from muse.quantities import production_amplitude + from muse.utilities import broadcast_techs - result = production_amplitude(production, technologies) + techs = broadcast_techs(technologies, production) + result = production_amplitude(production, techs) assert set(result.dims) == set(production.dims) - {"commodity"} From 6967f381f9d4ff381bf720c03cdfe64df8f3c4da Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 29 Jan 2025 10:13:14 +0000 Subject: [PATCH 06/12] Fix most remaining tests --- tests/conftest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ed793215c..dfa196ab1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -348,10 +348,12 @@ def create_agent(agent_args, technologies, stock, agent_type="retrofit") -> Agen # encompass every single technology. # This is not quite representative of the use case in the code, so in that # case, we add a bit of structure by removing some of the assets. - technology = set([u for u in technologies.technology.values]) - if set(agent.assets.technology.values) == technology: - techs = choice(technology, len(technology) // 2, replace=False) - agent.assets = agent.assets.sel(asset=techs) + technology_names = set(technologies.technology.values) + if set(agent.assets.technology.values) == technology_names: + techs = choice( + list(technology_names), len(technology_names) // 2, replace=False + ) + agent.assets = agent.assets.where(agent.assets.technology.isin(techs)) return agent From 0c24208994aaaca179c69c4a3542238b622f48b6 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 29 Jan 2025 11:43:29 +0000 Subject: [PATCH 07/12] Remove some unnecessary casting --- src/muse/quantities.py | 19 ++++++------------- src/muse/sectors/sector.py | 2 +- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/muse/quantities.py b/src/muse/quantities.py index 8eca0dc2c..11f197af9 100644 --- a/src/muse/quantities.py +++ b/src/muse/quantities.py @@ -9,8 +9,6 @@ from __future__ import annotations -from typing import cast - import numpy as np import xarray as xr @@ -143,9 +141,8 @@ def emission( from muse.utilities import broadcast_techs # just in case we are passed a technologies dataset, like in other functions - fouts = broadcast_techs( - getattr(fixed_outputs, "fixed_outputs", fixed_outputs), production - ) + fixed_outputs = getattr(fixed_outputs, "fixed_outputs", fixed_outputs) + fouts = broadcast_techs(fixed_outputs, production) envs = is_pollutant(fouts.comm_usage) enduses = is_enduse(fouts.comm_usage) return production.sel(commodity=enduses).sum("commodity") * broadcast_timeslice( @@ -270,8 +267,8 @@ def maximum_production( capa = filter_input( capacity, **{k: v for k, v in filters.items() if k in capacity.dims} ) - btechs = broadcast_techs( # type: ignore - cast(xr.Dataset, technologies[["fixed_outputs", "utilization_factor"]]), capa + btechs = broadcast_techs( + technologies[["fixed_outputs", "utilization_factor"]], capa ) ftechs = filter_input( btechs, **{k: v for k, v in filters.items() if k in btechs.dims} @@ -388,12 +385,8 @@ def minimum_production( if "minimum_service_factor" not in technologies: return broadcast_timeslice(xr.zeros_like(capa), level=timeslice_level) - btechs = broadcast_techs( # type: ignore - cast( - xr.Dataset, - technologies[["fixed_outputs", "minimum_service_factor"]], - ), - capa, + btechs = broadcast_techs( + technologies[["fixed_outputs", "minimum_service_factor"]], capa ) ftechs = filter_input( btechs, **{k: v for k, v in filters.items() if k in btechs.dims} diff --git a/src/muse/sectors/sector.py b/src/muse/sectors/sector.py index a8d44e33b..0bce7c2fb 100644 --- a/src/muse/sectors/sector.py +++ b/src/muse/sectors/sector.py @@ -314,7 +314,7 @@ def market_variables(self, market: xr.Dataset, technologies: xr.Dataset) -> Any: # Calculate LCOE # We select data for the second year, which corresponds to the investment year - technodata = cast(xr.Dataset, broadcast_techs(technologies, supply)) + technodata = broadcast_techs(technologies, supply) lcoe = levelized_cost_of_energy( prices=market.prices.sel(region=supply.region).isel(year=1), technologies=technodata, From 8096e661e13775fce3c02d5bc88b55c8c64ca43d Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 29 Jan 2025 12:51:46 +0000 Subject: [PATCH 08/12] Doctest, hardcode "asset" --- src/muse/utilities.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/muse/utilities.py b/src/muse/utilities.py index 9459196f9..8de3e1b49 100644 --- a/src/muse/utilities.py +++ b/src/muse/utilities.py @@ -182,7 +182,6 @@ def operation(x): def broadcast_techs( technologies: xr.Dataset | xr.DataArray, template: xr.DataArray | xr.Dataset, - dimension: str = "asset", interpolation: str = "linear", installed_as_year: bool = True, **kwargs, @@ -211,11 +210,41 @@ def broadcast_techs( applied to the `year` dimension of the technologies dataset kwargs: further arguments are used initial filters over the `technologies` dataset. + + Example: + Define the example array: + >>> import xarray as xr + >>> x = xr.DataArray( + ... data=[[1, 2, 3], [4, 5, 6]], + ... dims=['technology', 'region'], + ... coords={'technology': ['gasboiler', 'heatpump'], + ... 'region': ['R1', 'R2', 'R3']}, + ... ) + + Define the assets template: + >>> template = xr.DataArray( + ... data=[0, 0], + ... dims=["asset"], + ... coords={ + ... "region": (["asset"], ["R1", "R2"]), + ... "technology": (["asset"], ["gasboiler", "heatpump"]), + ... "installed": (["asset"], [2020, 2025])}, + ... ) + + Reshape/select the data to match the template: + >>> broadcast_techs(x, template) + Size: 16B + array([1, 5]) + Coordinates: + technology (asset) Date: Wed, 29 Jan 2025 12:58:03 +0000 Subject: [PATCH 09/12] Better docstring --- src/muse/utilities.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/muse/utilities.py b/src/muse/utilities.py index 8de3e1b49..69e478886 100644 --- a/src/muse/utilities.py +++ b/src/muse/utilities.py @@ -203,8 +203,6 @@ def broadcast_techs( Arguments: technologies: The dataset to broadcast template: the dataset or data-array to use as a template - dimension: the name of the dimensiom from `template` over which to - broadcast interpolation: interpolation method used across `year` installed_as_year: if the coordinate `installed` exists, then it is applied to the `year` dimension of the technologies dataset @@ -227,8 +225,7 @@ def broadcast_techs( ... dims=["asset"], ... coords={ ... "region": (["asset"], ["R1", "R2"]), - ... "technology": (["asset"], ["gasboiler", "heatpump"]), - ... "installed": (["asset"], [2020, 2025])}, + ... "technology": (["asset"], ["gasboiler", "heatpump"])}, ... ) Reshape/select the data to match the template: @@ -238,8 +235,11 @@ def broadcast_techs( Coordinates: technology (asset) Date: Wed, 29 Jan 2025 13:34:34 +0000 Subject: [PATCH 10/12] Fix remaining test (?) --- tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index dfa196ab1..7d270d46c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -336,10 +336,13 @@ def create_agent(agent_args, technologies, stock, agent_type="retrofit") -> Agen from muse.agents.factories import create_agent + region = agent_args["region"] agent = create_agent( agent_type=agent_type, - technologies=technologies, - capacity=stock, + technologies=technologies.sel(region=region), + capacity=stock.where(stock.region == region, drop=True).assign_coords( + region=region + ), year=2010, **agent_args, ) @@ -364,7 +367,6 @@ def newcapa_agent(agent_args, technologies, stock) -> Agent: @fixture def retro_agent(agent_args, technologies, stock) -> Agent: - agent_args["investment"] = "adhoc" # fails with scipy solver, see # 587 return create_agent(agent_args, technologies, stock.capacity, "retrofit") From a8f9e8222551b2cc310324b7a170365a1f86e57a Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 29 Jan 2025 15:09:14 +0000 Subject: [PATCH 11/12] Error message and better docstring --- src/muse/__main__.py | 11 ++++++++++- src/muse/utilities.py | 23 +++++++++++++++-------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/muse/__main__.py b/src/muse/__main__.py index e5854ba91..5ab9789cc 100644 --- a/src/muse/__main__.py +++ b/src/muse/__main__.py @@ -107,7 +107,16 @@ def patched_broadcast_compat_data(self, other): if "asset" in dims and any( dim in dims for dim in ["region", "technology", "installed"] ): - raise ValueError() + raise ValueError( + "DataArrays with an 'asset' dimension cannot be broadcasted along " + "'region', 'technology', or 'installed' dimensions. " + "This error is usually raised when attempting to combine asset-level data " + "(e.g. a capacity dataset with an 'asset' dimension) with a fully explicit " + "technology dataset (e.g. a technology dataset with 'region' and " + "'technology' dimensions). " + "Please use `broadcast_techs` on the latter object before performing this " + "operation." + ) return self_data, other_data, dims diff --git a/src/muse/utilities.py b/src/muse/utilities.py index 0c3ec5996..71661f8d7 100644 --- a/src/muse/utilities.py +++ b/src/muse/utilities.py @@ -210,25 +210,32 @@ def broadcast_techs( `technologies` dataset. Example: - Define the example array: + Define the technology array: >>> import xarray as xr - >>> x = xr.DataArray( + >>> technologies = xr.DataArray( ... data=[[1, 2, 3], [4, 5, 6]], ... dims=['technology', 'region'], ... coords={'technology': ['gasboiler', 'heatpump'], ... 'region': ['R1', 'R2', 'R3']}, ... ) + This array contains a value for every combination of technology and region (e.g. + this could refer to the efficiency of each technology in each region). Define the assets template: - >>> template = xr.DataArray( - ... data=[0, 0], + >>> assets = xr.DataArray( + ... data=[10, 50], ... dims=["asset"], ... coords={ ... "region": (["asset"], ["R1", "R2"]), ... "technology": (["asset"], ["gasboiler", "heatpump"])}, ... ) + We have two assets: a gas boiler in region R1 and a heat pump in region R2. In + this case the values don't matter, but could correspond to the installed + capacity of each asset, for example. - Reshape/select the data to match the template: + We want to select the values from the technology array that correspond to each + asset in the template. To do this, we perform `broadcast_techs` on + `technologies` using `assets` as a template: >>> broadcast_techs(x, template) Size: 16B array([1, 5]) @@ -237,9 +244,9 @@ def broadcast_techs( region (asset) Date: Wed, 29 Jan 2025 15:57:19 +0000 Subject: [PATCH 12/12] Fix doctest --- src/muse/utilities.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/muse/utilities.py b/src/muse/utilities.py index 71661f8d7..90ba49701 100644 --- a/src/muse/utilities.py +++ b/src/muse/utilities.py @@ -218,6 +218,7 @@ def broadcast_techs( ... coords={'technology': ['gasboiler', 'heatpump'], ... 'region': ['R1', 'R2', 'R3']}, ... ) + This array contains a value for every combination of technology and region (e.g. this could refer to the efficiency of each technology in each region). @@ -229,6 +230,7 @@ def broadcast_techs( ... "region": (["asset"], ["R1", "R2"]), ... "technology": (["asset"], ["gasboiler", "heatpump"])}, ... ) + We have two assets: a gas boiler in region R1 and a heat pump in region R2. In this case the values don't matter, but could correspond to the installed capacity of each asset, for example. @@ -236,7 +238,7 @@ def broadcast_techs( We want to select the values from the technology array that correspond to each asset in the template. To do this, we perform `broadcast_techs` on `technologies` using `assets` as a template: - >>> broadcast_techs(x, template) + >>> broadcast_techs(technologies, assets) Size: 16B array([1, 5]) Coordinates: