diff --git a/src/commodity.rs b/src/commodity.rs index d19050393..729a146dc 100644 --- a/src/commodity.rs +++ b/src/commodity.rs @@ -24,7 +24,7 @@ pub type DemandMap = HashMap<(RegionID, u32, TimeSliceSelection), Flow>; /// /// Represents a substance (e.g. CO2) or form of energy (e.g. electricity) that can be produced or /// consumed by processes. -#[derive(PartialEq, Debug, Deserialize)] +#[derive(PartialEq, Debug, Deserialize, Clone)] pub struct Commodity { /// Unique identifier for the commodity (e.g. "ELC") pub id: CommodityID, @@ -79,7 +79,7 @@ pub struct CommodityLevy { } /// Commodity balance type -#[derive(PartialEq, Debug, DeserializeLabeledStringEnum)] +#[derive(PartialEq, Debug, DeserializeLabeledStringEnum, Clone)] pub enum CommodityType { /// Supply and demand of this commodity must be balanced #[string = "sed"] diff --git a/src/graph.rs b/src/graph.rs index fe7ab3aee..516c6c39d 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,18 +1,66 @@ //! Module for creating and analysing commodity graphs -use crate::commodity::CommodityID; +use crate::commodity::{CommodityID, CommodityMap, CommodityType}; use crate::process::{ProcessID, ProcessMap}; use crate::region::RegionID; -use anyhow::{anyhow, Result}; +use crate::time_slice::{TimeSliceInfo, TimeSliceLevel, TimeSliceSelection}; +use crate::units::{Dimensionless, Flow}; +use anyhow::{anyhow, ensure, Context, Result}; +use indexmap::IndexSet; +use itertools::{iproduct, Itertools}; use petgraph::algo::toposort; use petgraph::graph::Graph; use petgraph::Directed; use std::collections::HashMap; +use std::fmt::Display; /// A graph of commodity flows for a given region and year -type CommoditiesGraph = Graph; +type CommoditiesGraph = Graph; -/// Creates a graph of commodity flows for a given region and year -pub fn create_commodities_graph_for_region_year( +#[derive(Eq, PartialEq, Clone, Hash)] +/// A node in the commodity graph +enum GraphNode { + /// A node representing a commodity + Commodity(CommodityID), + /// A source node for processes that have no inputs + Source, + /// A sink node for processes that have no outputs + Sink, + /// A demand node for commodities with service demands + Demand, +} + +impl Display for GraphNode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + GraphNode::Commodity(id) => write!(f, "{id}"), + GraphNode::Source => write!(f, "SOURCE"), + GraphNode::Sink => write!(f, "SINK"), + GraphNode::Demand => write!(f, "DEMAND"), + } + } +} + +#[derive(Eq, PartialEq, Clone, Hash)] +/// An edge in the commodity graph +enum GraphEdge { + /// An edge representing a process + Process(ProcessID), + /// An edge representing a service demand + Demand, +} + +/// Creates a directed graph of commodity flows for a given region and year. +/// +/// The graph contains nodes for all commodities that may be consumed/produced by processes in the +/// specified region/year. There will be an edge from commodity A to B if there exists a process +/// that consumes A and produces B. +/// +/// There are also special `Source` and `Sink` nodes, which are used for processes that have no +/// inputs or outputs. +/// +/// The graph does not take into account process availabilities or commodity demands, both of which +/// can vary by time slice. See `prepare_commodities_graph_for_validation`. +fn create_commodities_graph_for_region_year( processes: &ProcessMap, region_id: &RegionID, year: u32, @@ -22,40 +70,213 @@ pub fn create_commodities_graph_for_region_year( let key = (region_id.clone(), year); for process in processes.values() { - let Some(primary_output) = &process.primary_output else { - // Process only has input flows - continue; - }; - let Some(flows) = process.flows.get(&key) else { // Process doesn't operate in this region/year continue; }; - // Get input flows for the process - let inputs = flows + // Get output nodes for the process + let mut outputs: Vec<_> = flows + .values() + .filter(|flow| flow.is_output()) + .map(|flow| GraphNode::Commodity(flow.commodity.id.clone())) + .collect(); + + // Get input nodes for the process + let mut inputs: Vec<_> = flows .values() .filter(|flow| flow.is_input()) - .map(|flow| &flow.commodity.id); + .map(|flow| GraphNode::Commodity(flow.commodity.id.clone())) + .collect(); + + // Use `Source` node if no inputs, `Sink` node if no outputs + if inputs.is_empty() { + inputs.push(GraphNode::Source); + } + if outputs.is_empty() { + outputs.push(GraphNode::Sink); + } - // Create edges from inputs to primary outputs - // We also create nodes for commodities the first time they are encountered - for input in inputs { - let source_node = *commodity_to_node_index + // Create edges from all inputs to all outputs + // We also create nodes the first time they are encountered + for (input, output) in iproduct!(inputs, outputs) { + let source_node_index = *commodity_to_node_index .entry(input.clone()) .or_insert_with(|| graph.add_node(input.clone())); - let target_node = *commodity_to_node_index - .entry(primary_output.clone()) - .or_insert_with(|| graph.add_node(primary_output.clone())); - graph.add_edge(source_node, target_node, process.id.clone()); + let target_node_index = *commodity_to_node_index + .entry(output.clone()) + .or_insert_with(|| graph.add_node(output.clone())); + graph.add_edge( + source_node_index, + target_node_index, + GraphEdge::Process(process.id.clone()), + ); } } graph } -/// Performs topological sort on the commodity graph -pub fn topo_sort_commodities(graph: &CommoditiesGraph) -> Result> { +/// Prepares a graph for validation with `validate_commodities_graph`. +/// +/// It takes a base graph produced by `create_commodities_graph_for_region_year`, and modifies it to +/// account for process availabilities and commodity demands within the given time slice selection, +/// returning a new graph. +/// +/// Commodity demands are represented by the `Demand` node. We only add edges to the `Demand` node +/// for commodities with the same time_slice_level as the selection. Other demands can be ignored +/// since this graph will only be validated for commodities with the same time_slice_level as the +/// selection. +fn prepare_commodities_graph_for_validation( + base_graph: &CommoditiesGraph, + processes: &ProcessMap, + commodities: &CommodityMap, + time_slice_info: &TimeSliceInfo, + region_id: &RegionID, + year: u32, + time_slice_selection: &TimeSliceSelection, +) -> CommoditiesGraph { + let mut filtered_graph = base_graph.clone(); + + // Filter by process availability + // We keep edges if the process has availability > 0 in any time slice in the selection + filtered_graph.retain_edges(|graph, edge_idx| { + // Get the process for the edge + let GraphEdge::Process(process_id) = graph.edge_weight(edge_idx).unwrap() else { + panic!("Demand edges should not be present in the base graph"); + }; + let process = &processes[process_id]; + + // Check if the process has availability > 0 in any time slice in the selection + time_slice_selection + .iter(time_slice_info) + .any(|(time_slice, _)| { + let key = (region_id.clone(), year, time_slice.clone()); + process + .activity_limits + .get(&key) + .is_some_and(|avail| *avail.end() > Dimensionless(0.0)) + }) + }); + + // Add demand edges + // We add edges to the `Demand` node for commodities that are demanded in the selection + // NOTE: we only do this for commodities with the same time_slice_level as the selection + let demand_node_index = filtered_graph.add_node(GraphNode::Demand); + for (commodity_id, commodity) in commodities { + if time_slice_selection.level() == commodity.time_slice_level + && commodity + .demand + .get(&(region_id.clone(), year, time_slice_selection.clone())) + .is_some_and(|&v| v > Flow(0.0)) + { + let commodity_node = GraphNode::Commodity(commodity_id.clone()); + let commodity_node_index = filtered_graph + .node_indices() + .find(|&idx| filtered_graph.node_weight(idx) == Some(&commodity_node)) + .unwrap_or_else(|| { + filtered_graph.add_node(GraphNode::Commodity(commodity_id.clone())) + }); + filtered_graph.add_edge(commodity_node_index, demand_node_index, GraphEdge::Demand); + } + } + + filtered_graph +} + +/// Validates that the commodity graph follows the rules for different commodity types. +/// +/// It takes as input a graph created by `create_commodities_graph_for_validation`, which is built +/// for a specific time slice selection (must match the `time_slice_level` passed to this function). +/// +/// The validation is only performed for commodities with the specified time slice level. For full +/// validation of all commodities in the model, we therefore need to run this function for all time +/// slice selections at all time slice levels. This is handled by +/// `build_and_validate_commodity_graphs_for_model`. +fn validate_commodities_graph( + graph: &CommoditiesGraph, + commodities: &CommodityMap, + time_slice_level: TimeSliceLevel, +) -> Result<()> { + for node_idx in graph.node_indices() { + // Get the commodity ID for the node + let graph_node = graph.node_weight(node_idx).unwrap(); + let commodity_id = match graph_node { + GraphNode::Commodity(id) => id, + // Skip special nodes + _ => continue, + }; + + // Only validate commodities with the specified time slice level + let commodity = &commodities[commodity_id]; + if commodity.time_slice_level != time_slice_level { + continue; + } + + // Count the incoming and outgoing edges for the commodity + let has_incoming = graph + .edges_directed(node_idx, petgraph::Direction::Incoming) + .next() + .is_some(); + let has_outgoing = graph + .edges_directed(node_idx, petgraph::Direction::Outgoing) + .next() + .is_some(); + + // Match validation rules to commodity type + match commodity.kind { + CommodityType::ServiceDemand => { + // Cannot have outgoing `Process` (non-`Demand`) edges + let has_non_demand_outgoing = graph + .edges_directed(node_idx, petgraph::Direction::Outgoing) + .any(|edge| edge.weight() != &GraphEdge::Demand); + ensure!( + !has_non_demand_outgoing, + "SVD commodity {} cannot be an input to a process", + commodity_id + ); + + // If it has `Demand` edges, it must have at least one producer + let has_demand_edges = graph + .edges_directed(node_idx, petgraph::Direction::Outgoing) + .any(|edge| edge.weight() == &GraphEdge::Demand); + if has_demand_edges { + ensure!( + has_incoming, + "SVD commodity {} is demanded but has no producers", + commodity_id + ); + } + } + CommodityType::SupplyEqualsDemand => { + // SED: if consumed (outgoing edges), must also be produced (incoming edges) + ensure!( + !has_outgoing || has_incoming, + "SED commodity {} may be consumed but has no producers", + commodity_id + ); + } + CommodityType::Other => { + // OTH: cannot have both incoming and outgoing edges + ensure!( + !(has_incoming && has_outgoing), + "OTH commodity {} cannot have both producers and consumers", + commodity_id + ); + } + } + } + + Ok(()) +} + +/// Performs topological sort on the commodity graph to get the ordering for investments +/// +/// The returned Vec only includes SVD and SED commodities. +fn topo_sort_commodities( + graph: &CommoditiesGraph, + commodities: &CommodityMap, +) -> Result> { // Perform a topological sort on the graph let order = toposort(graph, None).map_err(|cycle| { let cycle_commodity = graph.node_weight(cycle.node_id()).unwrap().clone(); @@ -66,63 +287,328 @@ pub fn topo_sort_commodities(graph: &CommoditiesGraph) -> Result, + years: &[u32], + time_slice_info: &TimeSliceInfo, +) -> Result>> { + // Build base commodity graphs for each region and year + // These do not take into account demand and process availability + let commodity_graphs: HashMap<(RegionID, u32), CommoditiesGraph> = + iproduct!(region_ids, years.iter()) + .map(|(region_id, year)| { + let graph = create_commodities_graph_for_region_year(processes, region_id, *year); + ((region_id.clone(), *year), graph) + }) + .collect(); + + // Determine commodity ordering for each region and year + let commodity_order: HashMap<(RegionID, u32), Vec> = commodity_graphs + .iter() + .map(|((region_id, year), graph)| -> Result<_> { + let order = topo_sort_commodities(graph, commodities).with_context(|| { + format!("Error validating commodity graph for {region_id} in {year}") + })?; + Ok(((region_id.clone(), *year), order)) + }) + .try_collect()?; + + // Validate graphs at all time slice levels (taking into account process availability and demand) + for ((region_id, year), base_graph) in &commodity_graphs { + // Annual validation + let annual_graph = prepare_commodities_graph_for_validation( + base_graph, + processes, + commodities, + time_slice_info, + region_id, + *year, + &TimeSliceSelection::Annual, + ); + validate_commodities_graph(&annual_graph, commodities, TimeSliceLevel::Annual) + .with_context(|| { + format!("Error validating commodity graph for {region_id} in {year}") + })?; + + // Seasonal validation + for season in time_slice_info.iter_selections_at_level(TimeSliceLevel::Season) { + let seasonal_graph = prepare_commodities_graph_for_validation( + base_graph, + processes, + commodities, + time_slice_info, + region_id, + *year, + &season, + ); + validate_commodities_graph(&seasonal_graph, commodities, TimeSliceLevel::Season) + .with_context(|| { + format!( + "Error validating commodity graph for {region_id} in {year} in {season}" + ) + })?; + } + + // DayNight validation + for time_slice in time_slice_info.iter_selections_at_level(TimeSliceLevel::DayNight) { + let daynight_graph = prepare_commodities_graph_for_validation( + base_graph, + processes, + commodities, + time_slice_info, + region_id, + *year, + &time_slice, + ); + validate_commodities_graph(&daynight_graph, commodities, TimeSliceLevel::DayNight) + .with_context(|| { + format!( + "Error validating commodity graph for {region_id} in {year} in {time_slice}" + ) + })?; + } + } + + // If all the validation passes, return the commodity ordering + Ok(commodity_order) +} + #[cfg(test)] mod tests { use super::*; + use crate::commodity::Commodity; + use crate::fixture::{assert_error, other_commodity, sed_commodity, svd_commodity}; use petgraph::graph::Graph; + use rstest::rstest; + use std::rc::Rc; - #[test] - fn test_topo_sort_linear_graph() { + #[rstest] + fn test_topo_sort_linear_graph(sed_commodity: Commodity, svd_commodity: Commodity) { // Create a simple linear graph: A -> B -> C let mut graph = Graph::new(); - let node_a = graph.add_node(CommodityID::from("A")); - let node_b = graph.add_node(CommodityID::from("B")); - let node_c = graph.add_node(CommodityID::from("C")); + let node_a = graph.add_node(GraphNode::Commodity("A".into())); + let node_b = graph.add_node(GraphNode::Commodity("B".into())); + let node_c = graph.add_node(GraphNode::Commodity("C".into())); // Add edges: A -> B -> C - graph.add_edge(node_a, node_b, ProcessID::from("process1")); - graph.add_edge(node_b, node_c, ProcessID::from("process2")); + graph.add_edge(node_a, node_b, GraphEdge::Process("process1".into())); + graph.add_edge(node_b, node_c, GraphEdge::Process("process2".into())); + + // Create commodities map using fixtures + let mut commodities = CommodityMap::new(); + commodities.insert("A".into(), Rc::new(sed_commodity.clone())); + commodities.insert("B".into(), Rc::new(sed_commodity)); + commodities.insert("C".into(), Rc::new(svd_commodity)); - let result = topo_sort_commodities(&graph).unwrap(); + let result = topo_sort_commodities(&graph, &commodities).unwrap(); // Expected order: C, B, A (leaf nodes first) assert_eq!(result.len(), 3); - assert_eq!(result[0], CommodityID::from("C")); - assert_eq!(result[1], CommodityID::from("B")); - assert_eq!(result[2], CommodityID::from("A")); + assert_eq!(result[0], "C".into()); + assert_eq!(result[1], "B".into()); + assert_eq!(result[2], "A".into()); } - #[test] - fn test_topo_sort_cyclic_graph() { + #[rstest] + fn test_topo_sort_cyclic_graph(sed_commodity: Commodity) { // Create a simple cyclic graph: A -> B -> A let mut graph = Graph::new(); - let node_a = graph.add_node(CommodityID::from("A")); - let node_b = graph.add_node(CommodityID::from("B")); + let node_a = graph.add_node(GraphNode::Commodity("A".into())); + let node_b = graph.add_node(GraphNode::Commodity("B".into())); // Add edges creating a cycle: A -> B -> A - graph.add_edge(node_a, node_b, ProcessID::from("process1")); - graph.add_edge(node_b, node_a, ProcessID::from("process2")); + graph.add_edge(node_a, node_b, GraphEdge::Process("process1".into())); + graph.add_edge(node_b, node_a, GraphEdge::Process("process2".into())); - // This should return an error due to the cycle - let result = topo_sort_commodities(&graph); - assert!(result.is_err()); + // Create commodities map using fixtures + let mut commodities = CommodityMap::new(); + commodities.insert("A".into(), Rc::new(sed_commodity.clone())); + commodities.insert("B".into(), Rc::new(sed_commodity)); + // This should return an error due to the cycle // The error message should flag commodity B // Note: A is also involved in the cycle, but B is flagged as it is encountered first - let error_msg = result.unwrap_err().to_string(); - assert_eq!( - error_msg, - "Cycle detected in commodity graph for commodity B" + let result = topo_sort_commodities(&graph, &commodities); + assert_error!(result, "Cycle detected in commodity graph for commodity B"); + } + + #[rstest] + fn test_validate_commodities_graph( + other_commodity: Commodity, + sed_commodity: Commodity, + svd_commodity: Commodity, + ) { + let mut graph = Graph::new(); + let mut commodities = CommodityMap::new(); + + // Add test commodities (all have DayNight time slice level) + commodities.insert("A".into(), Rc::new(other_commodity)); + commodities.insert("B".into(), Rc::new(sed_commodity)); + commodities.insert("C".into(), Rc::new(svd_commodity)); + + // Build valid graph: A(OTH) -> B(SED) -> C(SVD) ->D(DEMAND) + let node_a = graph.add_node(GraphNode::Commodity("A".into())); + let node_b = graph.add_node(GraphNode::Commodity("B".into())); + let node_c = graph.add_node(GraphNode::Commodity("C".into())); + let node_d = graph.add_node(GraphNode::Demand); + graph.add_edge(node_a, node_b, GraphEdge::Process("process1".into())); + graph.add_edge(node_b, node_c, GraphEdge::Process("process2".into())); + graph.add_edge(node_c, node_d, GraphEdge::Demand); + + // Validate the graph at DayNight level + let result = validate_commodities_graph(&graph, &commodities, TimeSliceLevel::Annual); + assert!(result.is_ok()); + } + + #[rstest] + fn test_validate_commodities_graph_invalid_svd_consumed( + svd_commodity: Commodity, + sed_commodity: Commodity, + other_commodity: Commodity, + ) { + let mut graph = Graph::new(); + let mut commodities = CommodityMap::new(); + + // Add test commodities (all have DayNight time slice level) + commodities.insert("A".into(), Rc::new(svd_commodity)); + commodities.insert("B".into(), Rc::new(sed_commodity)); + commodities.insert("C".into(), Rc::new(other_commodity)); + + // Build invalid graph: C(OTH) -> A(SVD) -> B(SED) - SVD cannot be consumed + let node_c = graph.add_node(GraphNode::Commodity("C".into())); + let node_a = graph.add_node(GraphNode::Commodity("A".into())); + let node_b = graph.add_node(GraphNode::Commodity("B".into())); + graph.add_edge(node_c, node_a, GraphEdge::Process("process1".into())); + graph.add_edge(node_a, node_b, GraphEdge::Process("process2".into())); + + // Validate the graph at DayNight level + let result = validate_commodities_graph(&graph, &commodities, TimeSliceLevel::DayNight); + assert_error!(result, "SVD commodity A cannot be an input to a process"); + } + + #[rstest] + fn test_validate_commodities_graph_invalid_svd_not_produced(svd_commodity: Commodity) { + let mut graph = Graph::new(); + let mut commodities = CommodityMap::new(); + + // Add test commodities (all have DayNight time slice level) + commodities.insert("A".into(), Rc::new(svd_commodity)); + + // Build invalid graph: A(SVD) -> B(DEMAND) - SVD must be produced + let node_a = graph.add_node(GraphNode::Commodity("A".into())); + let node_b = graph.add_node(GraphNode::Demand); + graph.add_edge(node_a, node_b, GraphEdge::Demand); + + // Validate the graph at DayNight level + let result = validate_commodities_graph(&graph, &commodities, TimeSliceLevel::DayNight); + assert_error!(result, "SVD commodity A is demanded but has no producers"); + } + + #[rstest] + fn test_validate_commodities_graph_invalid_sed(sed_commodity: Commodity) { + let mut graph = Graph::new(); + let mut commodities = CommodityMap::new(); + + // Add test commodities (all have DayNight time slice level) + commodities.insert("A".into(), Rc::new(sed_commodity.clone())); + commodities.insert("B".into(), Rc::new(sed_commodity)); + + // Build invalid graph: B(SED) -> A(SED) + let node_a = graph.add_node(GraphNode::Commodity("A".into())); + let node_b = graph.add_node(GraphNode::Commodity("B".into())); + graph.add_edge(node_b, node_a, GraphEdge::Process("process1".into())); + + // Validate the graph at DayNight level + let result = validate_commodities_graph(&graph, &commodities, TimeSliceLevel::DayNight); + assert_error!( + result, + "SED commodity B may be consumed but has no producers" + ); + } + + #[rstest] + fn test_validate_commodities_graph_invalid_oth( + other_commodity: Commodity, + sed_commodity: Commodity, + ) { + let mut graph = Graph::new(); + let mut commodities = CommodityMap::new(); + + // Add test commodities (all have DayNight time slice level) + commodities.insert("A".into(), Rc::new(other_commodity)); + commodities.insert("B".into(), Rc::new(sed_commodity.clone())); + commodities.insert("C".into(), Rc::new(sed_commodity)); + + // Build invalid graph: B(SED) -> A(OTH) -> C(SED) + let node_a = graph.add_node(GraphNode::Commodity("A".into())); + let node_b = graph.add_node(GraphNode::Commodity("B".into())); + let node_c = graph.add_node(GraphNode::Commodity("C".into())); + graph.add_edge(node_b, node_a, GraphEdge::Process("process1".into())); + graph.add_edge(node_a, node_c, GraphEdge::Process("process2".into())); + + // Validate the graph at DayNight level + let result = validate_commodities_graph(&graph, &commodities, TimeSliceLevel::DayNight); + assert_error!( + result, + "OTH commodity A cannot have both producers and consumers" ); } } diff --git a/src/input.rs b/src/input.rs index a4a3210a7..90d00ab36 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1,13 +1,13 @@ //! Common routines for handling input data. use crate::asset::AssetPool; -use crate::graph::{create_commodities_graph_for_region_year, topo_sort_commodities}; +use crate::graph::build_and_validate_commodity_graphs_for_model; use crate::id::{HasID, IDLike}; use crate::model::{Model, ModelFile}; -use crate::units::{Dimensionless, UnitType}; +use crate::units::UnitType; use anyhow::{bail, ensure, Context, Result}; use float_cmp::approx_eq; use indexmap::IndexMap; -use itertools::{iproduct, Itertools}; +use itertools::Itertools; use serde::de::{Deserialize, DeserializeOwned, Deserializer}; use std::collections::{HashMap, HashSet}; use std::fs; @@ -200,15 +200,15 @@ pub fn load_model>(model_dir: P) -> Result<(Model, AssetPool)> { let agent_ids = agents.keys().cloned().collect(); let assets = read_assets(model_dir.as_ref(), &agent_ids, &processes, ®ion_ids)?; - // Determine commodity ordering for each region and year - let commodity_order = iproduct!(region_ids, years.iter()) - .map(|(region_id, year)| -> Result<_> { - let graph = create_commodities_graph_for_region_year(&processes, ®ion_id, *year); - let order = topo_sort_commodities(&graph) - .with_context(|| format!("Error with commodity graph for {region_id} in {year}"))?; - Ok(((region_id, *year), order)) - }) - .try_collect()?; + // Build and validate commodity graphs for all regions and years + // This gives us the commodity order for each region/year which is passed to the model + let commodity_order = build_and_validate_commodity_graphs_for_model( + &processes, + &commodities, + ®ion_ids, + years, + &time_slice_info, + )?; let model_path = model_dir .as_ref() @@ -231,6 +231,7 @@ pub fn load_model>(model_dir: P) -> Result<(Model, AssetPool)> { mod tests { use super::*; use crate::id::GenericID; + use crate::units::Dimensionless; use rstest::rstest; use serde::de::value::{Error as ValueError, F64Deserializer}; use serde::de::IntoDeserializer; diff --git a/src/input/process.rs b/src/input/process.rs index fa4f4d640..4059dae9f 100644 --- a/src/input/process.rs +++ b/src/input/process.rs @@ -1,18 +1,16 @@ //! Code for reading process-related information from CSV files. use super::*; -use crate::commodity::{Commodity, CommodityID, CommodityMap, CommodityType}; +use crate::commodity::CommodityMap; use crate::id::IDCollection; use crate::process::{ Process, ProcessActivityLimitsMap, ProcessFlowsMap, ProcessID, ProcessMap, ProcessParameterMap, }; use crate::region::{parse_region_str, RegionID}; -use crate::time_slice::{TimeSliceInfo, TimeSliceSelection}; -use crate::units::Flow; +use crate::time_slice::TimeSliceInfo; use anyhow::{ensure, Context, Ok, Result}; use indexmap::IndexSet; -use itertools::{chain, iproduct}; +use itertools::chain; use serde::Deserialize; -use std::collections::HashMap; use std::path::Path; use std::rc::Rc; @@ -62,16 +60,6 @@ pub fn read_processes( let mut flows = read_process_flows(model_dir, &mut processes, commodities)?; let mut parameters = read_process_parameters(model_dir, &processes, milestone_years[0])?; - // Validate commodities after the flows have been read - validate_commodities( - commodities, - &flows, - &activity_limits, - region_ids, - milestone_years, - time_slice_info, - )?; - // Add data to Process objects for (id, process) in processes.iter_mut() { // This will always succeed as we know there will only be one reference to the process here @@ -164,467 +152,3 @@ where Ok(processes) } - -/// Perform consistency checks for commodity flows. -fn validate_commodities( - commodities: &CommodityMap, - flows: &HashMap, - availabilities: &HashMap, - region_ids: &IndexSet, - milestone_years: &[u32], - time_slice_info: &TimeSliceInfo, -) -> Result<()> { - for commodity in commodities.values() { - if commodity.kind == CommodityType::Other { - validate_other_commodity(&commodity.id, flows)?; - continue; - } - - for (region_id, year) in iproduct!(region_ids.iter(), milestone_years.iter().copied()) { - match commodity.kind { - CommodityType::SupplyEqualsDemand => { - validate_sed_commodity(&commodity.id, flows, region_id, year)?; - } - CommodityType::ServiceDemand => { - for ts_selection in - time_slice_info.iter_selections_at_level(commodity.time_slice_level) - { - validate_svd_commodity( - time_slice_info, - commodity, - flows, - availabilities, - region_id, - year, - &ts_selection, - )?; - } - } - _ => unreachable!(), - } - } - } - - Ok(()) -} - -/// Check that commodities of type other are either produced or consumed but not both -fn validate_other_commodity( - commodity_id: &CommodityID, - flows: &HashMap, -) -> Result<()> { - let mut is_producer = None; - for flows in flows.values().flat_map(|flows| flows.values()) { - if let Some(flow) = flows.get(commodity_id) { - let cur_is_producer = flow.is_output(); - if let Some(is_producer) = is_producer { - ensure!( - is_producer == cur_is_producer, - "{commodity_id} is both a producer and consumer. \ - Commodities of type 'other' must only be consumed or produced." - ); - } else { - is_producer = Some(cur_is_producer); - } - } - } - - ensure!( - is_producer.is_some(), - "Commodity {commodity_id} is neither produced or consumed." - ); - - Ok(()) -} - -/// Check that an SED commodity has a consumer and producer process -fn validate_sed_commodity( - commodity_id: &CommodityID, - flows: &HashMap, - region_id: &RegionID, - year: u32, -) -> Result<()> { - let mut has_producer = false; - let mut has_consumer = false; - for flows in flows.values() { - let flows = flows.get(&(region_id.clone(), year)).unwrap(); - if let Some(flow) = flows.get(&commodity_id.clone()) { - if flow.is_output() { - has_producer = true; - } else if flow.is_input() { - has_consumer = true; - } - } - } - - ensure!( - has_consumer && has_producer, - "Commodity {} of 'SED' type must have both producer and consumer processes for region {} in year {}", - commodity_id, - region_id, - year, - ); - - Ok(()) -} - -fn validate_svd_commodity( - time_slice_info: &TimeSliceInfo, - commodity: &Commodity, - flows: &HashMap, - availabilities: &HashMap, - region_id: &RegionID, - year: u32, - ts_selection: &TimeSliceSelection, -) -> Result<()> { - // Check if the commodity has a demand in the given time slice, region and year. - // We only need to check for producers if there is positive demand. - let demand = *commodity - .demand - .get(&(region_id.clone(), year, ts_selection.clone())) - .unwrap(); - if demand <= Flow(0.0) { - return Ok(()); - } - - // We must check for producers in the given year, region and time slices. - // This includes checking if flow > 0 and if availability > 0. - for (process_id, flows) in flows.iter() { - let flows = flows.get(&(region_id.clone(), year)).unwrap(); - let Some(flow) = flows.get(&commodity.id) else { - // We're only interested in processes which produce this commodity - continue; - }; - ensure!( - flow.is_output(), - "SVD commodity {} is consumed by process {}. \ - SVD commodities can only be produced, not consumed.", - commodity.id, - process_id - ); - - // If the process has availability >0 in any time slice for this selection, we accept it - let availabilities = availabilities.get(process_id).unwrap(); - for (ts, _) in ts_selection.iter(time_slice_info) { - let availability = availabilities - .get(&(region_id.clone(), year, ts.clone())) - .unwrap(); - if *availability.end() > Dimensionless(0.0) { - return Ok(()); - } - } - } - - // If we reach this point it means there is no producer, so we return an error. - bail!( - "Commodity {} of 'SVD' type must have a producer process for region {} in year {} and time slice(s) {}", - commodity.id, - region_id, - year, - ts_selection, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::commodity::{CommodityLevyMap, DemandMap}; - use crate::fixture::{assert_error, time_slice, time_slice_info}; - use crate::process::{FlowType, ProcessFlow}; - use crate::time_slice::{TimeSliceID, TimeSliceLevel}; - use crate::units::{Dimensionless, FlowPerActivity, MoneyPerFlow}; - use indexmap::indexmap; - use rstest::{fixture, rstest}; - use std::iter; - - #[fixture] - fn commodity_sed() -> Commodity { - Commodity { - id: "commodity_sed".into(), - description: "SED commodity".into(), - kind: CommodityType::SupplyEqualsDemand, - time_slice_level: TimeSliceLevel::Annual, - levies: CommodityLevyMap::new(), - demand: DemandMap::new(), - } - } - - #[fixture] - fn input_flows_sed(commodity_sed: Commodity) -> ProcessFlowsMap { - ProcessFlowsMap::from_iter([( - ("GBR".into(), 2010), - indexmap! { commodity_sed.id.clone() => ProcessFlow { - commodity: commodity_sed.into(), - coeff: FlowPerActivity(-10.0), - kind: FlowType::Fixed, - cost: MoneyPerFlow(1.0), - }}, - )]) - } - - #[fixture] - fn output_flows_sed(commodity_sed: Commodity) -> ProcessFlowsMap { - ProcessFlowsMap::from_iter([( - ("GBR".into(), 2010), - indexmap! {commodity_sed.id.clone()=>ProcessFlow { - commodity: commodity_sed.into(), - coeff: FlowPerActivity(10.0), - kind: FlowType::Fixed, - cost: MoneyPerFlow(1.0), - }}, - )]) - } - - #[rstest] - fn test_validate_sed_commodity_valid( - commodity_sed: Commodity, - input_flows_sed: ProcessFlowsMap, - output_flows_sed: ProcessFlowsMap, - ) { - // Valid scenario - let flows = HashMap::from_iter([ - ("process1".into(), input_flows_sed.clone()), - ("process2".into(), output_flows_sed.clone()), - ]); - assert!(validate_sed_commodity(&commodity_sed.id, &flows, &"GBR".into(), 2010).is_ok()); - } - - #[rstest] - fn test_validate_sed_commodity_invalid_no_producer( - commodity_sed: Commodity, - input_flows_sed: ProcessFlowsMap, - ) { - // Invalid scenario: no producer - let flows = HashMap::from_iter([("process1".into(), input_flows_sed.clone())]); - assert_error!( - validate_sed_commodity(&commodity_sed.id, &flows, &"GBR".into(), 2010), - "Commodity commodity_sed of 'SED' type must have both producer and consumer processes for region GBR in year 2010" - ); - } - - #[rstest] - fn test_validate_sed_commodity(commodity_sed: Commodity, output_flows_sed: ProcessFlowsMap) { - // Invalid scenario: no consumer - let flows = HashMap::from_iter([("process2".into(), output_flows_sed.clone())]); - assert_error!( - validate_sed_commodity(&commodity_sed.id, &flows, &"GBR".into(), 2010), - "Commodity commodity_sed of 'SED' type must have both producer and consumer processes for region GBR in year 2010" - ); - } - - #[fixture] - fn commodity_svd(time_slice: TimeSliceID) -> Commodity { - let demand = DemandMap::from_iter([(("GBR".into(), 2010, time_slice.into()), Flow(10.0))]); - - Commodity { - id: "commodity_svd".into(), - description: "SVD commodity".into(), - kind: CommodityType::ServiceDemand, - time_slice_level: TimeSliceLevel::Annual, - levies: CommodityLevyMap::new(), - demand, - } - } - - #[fixture] - fn flows_svd(commodity_svd: Commodity) -> HashMap { - HashMap::from_iter([( - "process1".into(), - ProcessFlowsMap::from_iter([( - ("GBR".into(), 2010), - indexmap! { commodity_svd.id.clone() => ProcessFlow { - commodity: commodity_svd.into(), - coeff: FlowPerActivity(10.0), - kind: FlowType::Fixed, - cost: MoneyPerFlow(1.0), - }}, - )]), - )]) - } - - #[rstest] - fn test_validate_svd_commodity_valid( - commodity_svd: Commodity, - flows_svd: HashMap, - time_slice_info: TimeSliceInfo, - time_slice: TimeSliceID, - ) { - let availabilities = HashMap::from_iter([( - "process1".into(), - ProcessActivityLimitsMap::from_iter([( - ("GBR".into(), 2010, time_slice.clone()), - Dimensionless(0.1)..=Dimensionless(0.9), - )]), - )]); - - // Valid scenario - assert!(validate_svd_commodity( - &time_slice_info, - &commodity_svd, - &flows_svd, - &availabilities, - &"GBR".into(), - 2010, - &time_slice.into() - ) - .is_ok()); - } - - #[rstest] - fn test_validate_svd_commodity_invalid_no_availability( - time_slice_info: TimeSliceInfo, - commodity_svd: Commodity, - flows_svd: HashMap, - time_slice: TimeSliceID, - ) { - // Invalid scenario: no availability - let availabilities = HashMap::from_iter([( - "process1".into(), - ProcessActivityLimitsMap::from_iter([( - ("GBR".into(), 2010, time_slice.clone()), - Dimensionless(0.0)..=Dimensionless(0.0), - )]), - )]); - assert_error!( - validate_svd_commodity( - &time_slice_info, - &commodity_svd, - &flows_svd, - &availabilities, - &"GBR".into(), - 2010, - &time_slice.into() - ), - "Commodity commodity_svd of 'SVD' type must have a producer process \ - for region GBR in year 2010 and time slice(s) winter.day" - ); - } - - #[fixture] - fn commodity_other() -> Commodity { - Commodity { - id: "commodity_other".into(), - description: "Other commodity".into(), - kind: CommodityType::Other, - time_slice_level: TimeSliceLevel::Annual, - levies: CommodityLevyMap::new(), - demand: DemandMap::new(), - } - } - - #[fixture] - fn producer_flows(commodity_other: Commodity) -> ProcessFlowsMap { - ProcessFlowsMap::from_iter([( - ("GBR".into(), 2010), - indexmap! { commodity_other.id.clone() => ProcessFlow { - commodity: commodity_other.into(), - coeff: FlowPerActivity(10.0), - kind: FlowType::Fixed, - cost: MoneyPerFlow(1.0), - }}, - )]) - } - - #[fixture] - fn consumer_flows(commodity_other: Commodity) -> ProcessFlowsMap { - ProcessFlowsMap::from_iter([( - ("GBR".into(), 2010), - indexmap! { commodity_other.id.clone() => ProcessFlow { - commodity: commodity_other.into(), - coeff: FlowPerActivity(-10.0), - kind: FlowType::Fixed, - cost: MoneyPerFlow(1.0), - }}, - )]) - } - - #[rstest] - fn test_validate_other_commodity_valid_producer( - commodity_other: Commodity, - producer_flows: ProcessFlowsMap, - ) { - // Valid scenario: commodity is only produced - let flows = HashMap::from_iter([("process1".into(), producer_flows)]); - assert!(validate_other_commodity(&commodity_other.id, &flows).is_ok()); - } - - #[rstest] - fn test_validate_other_commodity_valid_consumer( - commodity_other: Commodity, - consumer_flows: ProcessFlowsMap, - ) { - // Valid scenario: commodity is only consumed - let flows = HashMap::from_iter([("process1".into(), consumer_flows)]); - assert!(validate_other_commodity(&commodity_other.id, &flows).is_ok()); - } - - #[rstest] - fn test_validate_other_commodity_invalid_both( - commodity_other: Commodity, - producer_flows: ProcessFlowsMap, - consumer_flows: ProcessFlowsMap, - ) { - // Invalid scenario: commodity is both produced and consumed - let flows = HashMap::from_iter([ - ("process1".into(), producer_flows), - ("process2".into(), consumer_flows), - ]); - assert_error!( - validate_other_commodity(&commodity_other.id, &flows), - "commodity_other is both a producer and consumer. \ - Commodities of type 'other' must only be consumed or produced." - ); - } - - #[rstest] - fn test_validate_other_commodity_invalid_neither(commodity_other: Commodity) { - // Invalid scenario: commodity is neither produced nor consumed - let flows = HashMap::new(); - assert_error!( - validate_other_commodity(&commodity_other.id, &flows), - "Commodity commodity_other is neither produced or consumed." - ); - } - - #[rstest] - fn test_validate_svd_commodity_invalid_consumed( - commodity_svd: Commodity, - time_slice_info: TimeSliceInfo, - time_slice: TimeSliceID, - ) { - let commodity_svd = Rc::new(commodity_svd); - let region_id = RegionID("GBR".into()); - let availabilities = HashMap::from_iter([( - "process1".into(), - ProcessActivityLimitsMap::from_iter([( - (region_id.clone(), 2010, time_slice.clone()), - Dimensionless(0.1)..=Dimensionless(0.9), - )]), - )]); - let flows = HashMap::from_iter(iter::once(( - "process1".into(), - ProcessFlowsMap::from_iter([( - (region_id.clone(), 2010), - indexmap! { commodity_svd.id.clone() => ProcessFlow { - commodity: Rc::clone(&commodity_svd), - coeff: FlowPerActivity(-10.0), - kind: FlowType::Fixed, - cost: MoneyPerFlow(1.0), - }}, - )]), - ))); - assert_error!( - validate_svd_commodity( - &time_slice_info, - &commodity_svd, - &flows, - &availabilities, - ®ion_id, - 2010, - &time_slice.into() - ), - "SVD commodity commodity_svd is consumed by process process1. \ - SVD commodities can only be produced, not consumed." - ); - } -}