From c7bf4f82bc65abcf9c5f652d00a2f69ab417693f Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 10 Nov 2025 13:48:00 +0000 Subject: [PATCH 1/4] Check flows in milestone years only --- src/input/process.rs | 2 +- src/input/process/flow.rs | 69 +++++++++++++++++++++++++++------------ 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/input/process.rs b/src/input/process.rs index 3352e7d8d..c95ba05a1 100644 --- a/src/input/process.rs +++ b/src/input/process.rs @@ -61,7 +61,7 @@ pub fn read_processes( let mut processes = read_processes_file(model_dir, milestone_years, region_ids, commodities)?; let mut activity_limits = read_process_availabilities(model_dir, &processes, time_slice_info, base_year)?; - let mut flows = read_process_flows(model_dir, &mut processes, commodities)?; + let mut flows = read_process_flows(model_dir, &mut processes, commodities, milestone_years)?; let mut parameters = read_process_parameters(model_dir, &processes, base_year)?; // Add data to Process objects diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs index a80f2e6cb..56b62ae09 100644 --- a/src/input/process/flow.rs +++ b/src/input/process/flow.rs @@ -11,7 +11,7 @@ use anyhow::{Context, Result, ensure}; use indexmap::IndexMap; use itertools::iproduct; use serde::Deserialize; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use std::rc::Rc; @@ -62,10 +62,11 @@ pub fn read_process_flows( model_dir: &Path, processes: &mut ProcessMap, commodities: &CommodityMap, + milestone_years: &[u32], ) -> Result> { let file_path = model_dir.join(PROCESS_FLOWS_FILE_NAME); let process_flow_csv = read_csv(&file_path)?; - read_process_flows_from_iter(process_flow_csv, processes, commodities) + read_process_flows_from_iter(process_flow_csv, processes, commodities, milestone_years) .with_context(|| input_err_msg(&file_path)) } @@ -74,6 +75,7 @@ fn read_process_flows_from_iter( iter: I, processes: &mut ProcessMap, commodities: &CommodityMap, + milestone_years: &[u32], ) -> Result> where I: Iterator, @@ -134,8 +136,8 @@ where } } - validate_flows_and_update_primary_output(processes, &flows_map)?; - validate_secondary_flows(processes, &flows_map)?; + validate_flows_and_update_primary_output(processes, &flows_map, milestone_years)?; + validate_secondary_flows(processes, &flows_map, milestone_years)?; Ok(flows_map) } @@ -143,29 +145,33 @@ where fn validate_flows_and_update_primary_output( processes: &mut ProcessMap, flows_map: &HashMap, + milestone_years: &[u32], ) -> Result<()> { for (process_id, process) in processes.iter_mut() { let map = flows_map .get(process_id) .with_context(|| format!("Missing flows map for process {process_id}"))?; + let region_year: Vec<(&RegionID, &u32)> = + iproduct!(process.regions.iter(), milestone_years.iter()).collect(); + ensure!( - map.len() == process.years.len() * process.regions.len(), - "Flows map for process {process_id} does not cover all regions and years" + region_year + .iter() + .all(|(region_id, year)| map.contains_key(&((*region_id).clone(), **year))), + "Flows map for process {process_id} does not cover all regions and milestone years" ); - let mut iter = iproduct!(process.years.iter(), process.regions.iter()); - let primary_output = if let Some(primary_output) = &process.primary_output { Some(primary_output.clone()) } else { - let (year, region_id) = iter.next().unwrap(); + let (region_id, year) = region_year[0]; infer_primary_output(&map[&(region_id.clone(), *year)]).with_context(|| { format!("Could not infer primary_output for process {process_id}") })? }; - for (&year, region_id) in iter { + for (region_id, &year) in region_year { let flows = &map[&(region_id.clone(), year)]; // Check that the process has flows for this region/year @@ -236,11 +242,12 @@ fn check_flows_primary_output( Ok(()) } -/// Checks that non-primary io are defined for all years (within a region) and that -/// they are only inputs or only outputs in all years. +/// Checks that non-primary io are defined for all milestone years, at least, (within a region) and +/// that they are only inputs or only outputs in all years. fn validate_secondary_flows( processes: &mut ProcessMap, flows_map: &HashMap, + milestone_years: &[u32], ) -> Result<()> { for (process_id, process) in processes.iter() { // Get the flows for this process - there should be no error, as was checked already @@ -251,7 +258,11 @@ fn validate_secondary_flows( // Get the non-primary io flows for all years, if any, arranged by (commodity, region) let iter = iproduct!(process.years.iter(), process.regions.iter()); let mut flows: HashMap<(CommodityID, RegionID), Vec<&ProcessFlow>> = HashMap::new(); + let mut years: HashMap<(CommodityID, RegionID), HashSet> = HashMap::new(); for (&year, region_id) in iter { + if !map.contains_key(&(region_id.clone(), year)) { + continue; + } let flow = map[&(region_id.clone(), year)] .iter() .filter_map(|(commodity_id, flow)| { @@ -260,17 +271,21 @@ fn validate_secondary_flows( }); for (key, value) in flow { - flows.entry(key).or_default().push(value); + flows.entry(key.clone()).or_default().push(value); + if milestone_years.contains(&year) { + years.entry(key).or_default().insert(year); + } } } // Finally we check that the flows for a given commodity and region are defined for all - // years and that they are all inputs or all outputs + // milestone years and that they are all inputs or all outputs. This later check is done + // for all years, milestone or not. for ((commodity_id, region_id), value) in &flows { ensure!( - value.len() == process.years.len(), + years[&(commodity_id.clone(), region_id.clone())].len() == milestone_years.len(), "Flow of commodity {commodity_id} in region {region_id} for process {process_id} \ - does not cover all years" + does not cover all milestone years" ); let input_or_zero = value .iter() @@ -331,12 +346,16 @@ mod tests { #[rstest] fn single_output_infer_primary(#[from(svd_commodity)] commodity: Commodity, process: Process) { + let milestone_years = vec![2010, 2020]; let commodity = Rc::new(commodity); let (mut processes, flows_map) = build_maps( process, std::iter::once((commodity.id.clone(), flow(commodity.clone(), 1.0))), ); - assert!(validate_flows_and_update_primary_output(&mut processes, &flows_map).is_ok()); + assert!( + validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) + .is_ok() + ); assert_eq!( processes.values().exactly_one().unwrap().primary_output, Some(commodity.id.clone()) @@ -349,6 +368,7 @@ mod tests { #[from(sed_commodity)] commodity2: Commodity, process: Process, ) { + let milestone_years = vec![2010, 2020]; let commodity1 = Rc::new(commodity1); let commodity2 = Rc::new(commodity2); let (mut processes, flows_map) = build_maps( @@ -359,7 +379,8 @@ mod tests { ] .into_iter(), ); - let res = validate_flows_and_update_primary_output(&mut processes, &flows_map); + let res = + validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years); assert_error!(res, "Could not infer primary_output for process process1"); } @@ -369,6 +390,7 @@ mod tests { #[from(sed_commodity)] commodity2: Commodity, process: Process, ) { + let milestone_years = vec![2010, 2020]; let commodity1 = Rc::new(commodity1); let commodity2 = Rc::new(commodity2); let mut process = process; @@ -381,7 +403,10 @@ mod tests { ] .into_iter(), ); - assert!(validate_flows_and_update_primary_output(&mut processes, &flows_map).is_ok()); + assert!( + validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) + .is_ok() + ); assert_eq!( processes.values().exactly_one().unwrap().primary_output, Some(commodity2.id.clone()) @@ -394,6 +419,7 @@ mod tests { #[from(sed_commodity)] commodity2: Commodity, process: Process, ) { + let milestone_years = vec![2010, 2020]; let commodity1 = Rc::new(commodity1); let commodity2 = Rc::new(commodity2); let (mut processes, flows_map) = build_maps( @@ -404,7 +430,10 @@ mod tests { ] .into_iter(), ); - assert!(validate_flows_and_update_primary_output(&mut processes, &flows_map).is_ok()); + assert!( + validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) + .is_ok() + ); assert_eq!( processes.values().exactly_one().unwrap().primary_output, None From 5c4cdac27cfda05459a763d1c7d83c2f26baa165 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 10 Nov 2025 13:55:25 +0000 Subject: [PATCH 2/4] Remove unnecessary check --- src/input/process/flow.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs index 56b62ae09..9700b2bde 100644 --- a/src/input/process/flow.rs +++ b/src/input/process/flow.rs @@ -260,9 +260,6 @@ fn validate_secondary_flows( let mut flows: HashMap<(CommodityID, RegionID), Vec<&ProcessFlow>> = HashMap::new(); let mut years: HashMap<(CommodityID, RegionID), HashSet> = HashMap::new(); for (&year, region_id) in iter { - if !map.contains_key(&(region_id.clone(), year)) { - continue; - } let flow = map[&(region_id.clone(), year)] .iter() .filter_map(|(commodity_id, flow)| { From 272820ce0d41180a013e4b8128486c1d60b10c4a Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 11 Nov 2025 06:18:12 +0000 Subject: [PATCH 3/4] Check only milestone y within process range of activity --- src/input/process/flow.rs | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs index 9700b2bde..5c977cffd 100644 --- a/src/input/process/flow.rs +++ b/src/input/process/flow.rs @@ -11,7 +11,7 @@ use anyhow::{Context, Result, ensure}; use indexmap::IndexMap; use itertools::iproduct; use serde::Deserialize; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::path::Path; use std::rc::Rc; @@ -152,14 +152,18 @@ fn validate_flows_and_update_primary_output( .get(process_id) .with_context(|| format!("Missing flows map for process {process_id}"))?; + // Flows are required for all milestone years within the process years of activity + let required_years = milestone_years + .iter() + .filter(|&y| process.years.contains(y)); let region_year: Vec<(&RegionID, &u32)> = - iproduct!(process.regions.iter(), milestone_years.iter()).collect(); + iproduct!(process.regions.iter(), required_years).collect(); ensure!( region_year .iter() .all(|(region_id, year)| map.contains_key(&((*region_id).clone(), **year))), - "Flows map for process {process_id} does not cover all regions and milestone years" + "Flows map for process {process_id} does not cover all regions and required years" ); let primary_output = if let Some(primary_output) = &process.primary_output { @@ -255,10 +259,16 @@ fn validate_secondary_flows( .get(process_id) .with_context(|| format!("Missing flows map for process {process_id}"))?; + // Flows are required for all milestone years within the process years of activity + let required_years: Vec<&u32> = milestone_years + .iter() + .filter(|&y| process.years.contains(y)) + .collect(); + // Get the non-primary io flows for all years, if any, arranged by (commodity, region) let iter = iproduct!(process.years.iter(), process.regions.iter()); let mut flows: HashMap<(CommodityID, RegionID), Vec<&ProcessFlow>> = HashMap::new(); - let mut years: HashMap<(CommodityID, RegionID), HashSet> = HashMap::new(); + let mut number_of_years: HashMap<(CommodityID, RegionID), u32> = HashMap::new(); for (&year, region_id) in iter { let flow = map[&(region_id.clone(), year)] .iter() @@ -269,20 +279,21 @@ fn validate_secondary_flows( for (key, value) in flow { flows.entry(key.clone()).or_default().push(value); - if milestone_years.contains(&year) { - years.entry(key).or_default().insert(year); + if required_years.contains(&&year) { + *number_of_years.entry(key).or_default() += 1; } } } // Finally we check that the flows for a given commodity and region are defined for all // milestone years and that they are all inputs or all outputs. This later check is done - // for all years, milestone or not. + // for all years in the process range, required or not. for ((commodity_id, region_id), value) in &flows { ensure!( - years[&(commodity_id.clone(), region_id.clone())].len() == milestone_years.len(), + number_of_years[&(commodity_id.clone(), region_id.clone())] + == required_years.len().try_into().unwrap(), "Flow of commodity {commodity_id} in region {region_id} for process {process_id} \ - does not cover all milestone years" + does not cover all milestone years within the process range of activity." ); let input_or_zero = value .iter() From 9136009751ba0cdc6083018e90208c94eb2a2614 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 11 Nov 2025 06:28:09 +0000 Subject: [PATCH 4/4] white_check_mark: Update tests to cover new functionality --- src/asset.rs | 4 +-- src/fixture.rs | 12 ++++++-- src/input/process/flow.rs | 59 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/asset.rs b/src/asset.rs index 6d40b9e8d..68f606a80 100644 --- a/src/asset.rs +++ b/src/asset.rs @@ -1023,8 +1023,8 @@ mod tests { let agent_id = AgentID("agent1".into()); let region_id = RegionID("GBR".into()); assert_error!( - Asset::new_future(agent_id, process.into(), region_id, Capacity(1.0), 2009), - "Process process1 does not operate in the year 2009" + Asset::new_future(agent_id, process.into(), region_id, Capacity(1.0), 2007), + "Process process1 does not operate in the year 2007" ); } diff --git a/src/fixture.rs b/src/fixture.rs index 051957eaa..b571cc22e 100644 --- a/src/fixture.rs +++ b/src/fixture.rs @@ -148,13 +148,19 @@ pub fn process( region_ids: IndexSet, process_parameter_map: ProcessParameterMap, ) -> Process { - let years = vec![2010, 2015, 2020]; + let milestone_years = vec![2010, 2015, 2020]; + // The process start year is before the base year + let years = vec![2008, 2009] + .iter() + .chain(&milestone_years) + .cloned() + .collect(); // Create maps with (empty) entries for every region/year combo - let activity_limits = iproduct!(region_ids.iter(), years.iter()) + let activity_limits = iproduct!(region_ids.iter(), milestone_years.iter()) .map(|(region_id, year)| ((region_id.clone(), *year), Rc::new(HashMap::new()))) .collect(); - let flows = iproduct!(region_ids.iter(), years.iter()) + let flows = iproduct!(region_ids.iter(), milestone_years.iter()) .map(|(region_id, year)| ((region_id.clone(), *year), Rc::new(IndexMap::new()))) .collect(); Process { diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs index 5c977cffd..9d58d6985 100644 --- a/src/input/process/flow.rs +++ b/src/input/process/flow.rs @@ -338,12 +338,14 @@ mod tests { fn build_maps( process: Process, flows: I, + years: Option<&Vec>, ) -> (ProcessMap, HashMap) where I: Clone + Iterator, { + let years = years.unwrap_or(&process.years); let map: Rc> = Rc::new(flows.clone().collect()); - let flows_inner = iproduct!(&process.regions, &process.years) + let flows_inner = iproduct!(&process.regions, years) .map(|(region_id, year)| ((region_id.clone(), *year), map.clone())) .collect(); let flows = hash_map! {process.id.clone() => flows_inner}; @@ -359,6 +361,7 @@ mod tests { let (mut processes, flows_map) = build_maps( process, std::iter::once((commodity.id.clone(), flow(commodity.clone(), 1.0))), + None, ); assert!( validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) @@ -386,6 +389,7 @@ mod tests { (commodity2.id.clone(), flow(commodity2.clone(), 2.0)), ] .into_iter(), + None, ); let res = validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years); @@ -410,6 +414,7 @@ mod tests { (commodity2.id.clone(), flow(commodity2.clone(), 2.0)), ] .into_iter(), + None, ); assert!( validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) @@ -437,6 +442,7 @@ mod tests { (commodity2.id.clone(), flow(commodity2.clone(), -2.0)), ] .into_iter(), + None, ); assert!( validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) @@ -447,4 +453,55 @@ mod tests { None ); } + + #[rstest] + fn flows_not_in_all_milestone_years( + #[from(svd_commodity)] commodity1: Commodity, + #[from(sed_commodity)] commodity2: Commodity, + process: Process, + ) { + let milestone_years = vec![2010, 2015, 2020]; + let flow_years = vec![2010, 2020]; + let commodity1 = Rc::new(commodity1); + let commodity2 = Rc::new(commodity2); + let (mut processes, flows_map) = build_maps( + process, + [ + (commodity1.id.clone(), flow(commodity1.clone(), 1.0)), + (commodity2.id.clone(), flow(commodity2.clone(), 2.0)), + ] + .into_iter(), + Some(&flow_years), + ); + let res = + validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years); + assert_error!( + res, + "Flows map for process process1 does not cover all regions and required years" + ); + } + + #[rstest] + fn flows_only_milestone_years( + #[from(svd_commodity)] commodity1: Commodity, + #[from(sed_commodity)] commodity2: Commodity, + process: Process, + ) { + let milestone_years = vec![2010, 2015, 2020]; + let commodity1 = Rc::new(commodity1); + let commodity2 = Rc::new(commodity2); + let (mut processes, flows_map) = build_maps( + process, + [ + (commodity1.id.clone(), flow(commodity1.clone(), 1.0)), + (commodity2.id.clone(), flow(commodity2.clone(), -2.0)), + ] + .into_iter(), + Some(&milestone_years), + ); + assert!( + validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) + .is_ok() + ); + } }