From 014bdad46b4177de9766ba73907a9edfdfd24699 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 21 Nov 2025 13:56:11 +0000 Subject: [PATCH 1/5] :sparkles: Make process comission years a range --- src/asset.rs | 6 ++--- src/fixture.rs | 6 +---- src/input/agent/search_space.rs | 2 +- src/input/process.rs | 15 +++--------- src/input/process/availability.rs | 12 ++++------ src/input/process/flow.rs | 34 +++++++++++++-------------- src/input/process/parameter.rs | 39 +++++++++++++++++-------------- src/process.rs | 6 ++--- 8 files changed, 52 insertions(+), 68 deletions(-) diff --git a/src/asset.rs b/src/asset.rs index 68f606a80..cfa305306 100644 --- a/src/asset.rs +++ b/src/asset.rs @@ -972,7 +972,7 @@ mod tests { parameters: process_parameter_map, regions: indexset! {region_id.clone()}, primary_output: Some(commodity_id.clone()), - years: vec![2020], + years: 2020..=2020, activity_limits, capacity_to_activity: ActivityPerCapacity(1.0), }); @@ -1063,7 +1063,7 @@ mod tests { let process = Rc::new(Process { id: "process1".into(), description: "Description".into(), - years: vec![2010, 2020], + years: 2010..=2020, activity_limits, flows, parameters: process_parameter_map, @@ -1118,7 +1118,7 @@ mod tests { Process { id: "process1".into(), description: "Description".into(), - years: vec![2010, 2020], + years: 2010..=2020, activity_limits, flows, parameters: process_parameter_map, diff --git a/src/fixture.rs b/src/fixture.rs index b571cc22e..bfd32d0d7 100644 --- a/src/fixture.rs +++ b/src/fixture.rs @@ -150,11 +150,7 @@ pub fn process( ) -> Process { let milestone_years = vec![2010, 2015, 2020]; // The process start year is before the base year - let years = vec![2008, 2009] - .iter() - .chain(&milestone_years) - .cloned() - .collect(); + let years = 2008..=*milestone_years.last().unwrap(); // Create maps with (empty) entries for every region/year combo let activity_limits = iproduct!(region_ids.iter(), milestone_years.iter()) diff --git a/src/input/agent/search_space.rs b/src/input/agent/search_space.rs index b31e4442b..3dbdf7940 100644 --- a/src/input/agent/search_space.rs +++ b/src/input/agent/search_space.rs @@ -219,7 +219,7 @@ mod tests { let process = Process { id: id.clone(), description: "Description".into(), - years: vec![2010, 2020], + years: 2010..=2020, activity_limits: ProcessActivityLimitsMap::new(), flows: ProcessFlowsMap::new(), parameters: ProcessParameterMap::new(), diff --git a/src/input/process.rs b/src/input/process.rs index c95ba05a1..9415a5930 100644 --- a/src/input/process.rs +++ b/src/input/process.rs @@ -10,7 +10,6 @@ use crate::time_slice::TimeSliceInfo; use crate::units::ActivityPerCapacity; use anyhow::{Context, Ok, Result, ensure}; use indexmap::IndexSet; -use itertools::chain; use serde::Deserialize; use std::path::Path; use std::rc::Rc; @@ -62,7 +61,7 @@ pub fn read_processes( let mut activity_limits = read_process_availabilities(model_dir, &processes, time_slice_info, base_year)?; let mut flows = read_process_flows(model_dir, &mut processes, commodities, milestone_years)?; - let mut parameters = read_process_parameters(model_dir, &processes, base_year)?; + let mut parameters = read_process_parameters(model_dir, &processes, milestone_years)?; // Add data to Process objects for (id, process) in &mut processes { @@ -114,16 +113,8 @@ where ); // Select process years. It is possible for assets to have been commissioned before the - // simulation's time horizon, so assume that all years >=start_year and =start_year are valid too. + let years = start_year..=end_year; // Parse region ID let regions = parse_region_str(&process_raw.regions, region_ids)?; diff --git a/src/input/process/availability.rs b/src/input/process/availability.rs index 61fe44002..87292ac53 100644 --- a/src/input/process/availability.rs +++ b/src/input/process/availability.rs @@ -137,9 +137,9 @@ where })?; // Get years - let process_years = &process.years; + let process_years: Vec = process.years.clone().collect(); let record_years = - parse_year_str(&record.commission_years, process_years).with_context(|| { + parse_year_str(&record.commission_years, &process_years).with_context(|| { format!("Invalid year for process {id}. Valid years are {process_years:?}") })?; @@ -197,11 +197,7 @@ fn check_missing_milestone_years( map_for_process: &ProcessActivityLimitsMap, base_year: u32, ) -> Result<()> { - let process_milestone_years = process - .years - .iter() - .copied() - .filter(|&year| year >= base_year); + let process_milestone_years = process.years.clone().filter(|&year| year >= base_year); let mut missing = Vec::new(); for (region_id, year) in iproduct!(&process.regions, process_milestone_years) { if !map_for_process.contains_key(&(region_id.clone(), year)) { @@ -227,7 +223,7 @@ fn check_missing_time_slices( time_slice_info: &TimeSliceInfo, ) -> Result<()> { let mut missing = Vec::new(); - for (region_id, &year) in iproduct!(&process.regions, &process.years) { + for (region_id, year) in iproduct!(&process.regions, process.years.clone()) { if let Some(map_for_region_year) = map_for_process.get(&(region_id.clone(), year)) { // There are at least some entries for this region/year combo; check if there are // any time slices not covered diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs index 9d58d6985..04e31825c 100644 --- a/src/input/process/flow.rs +++ b/src/input/process/flow.rs @@ -97,9 +97,9 @@ where })?; // Get years - let process_years = &process.years; + let process_years: Vec = process.years.clone().collect(); let record_years = - parse_year_str(&record.commission_years, process_years).with_context(|| { + parse_year_str(&record.commission_years, &process_years).with_context(|| { format!("Invalid year for process {id}. Valid years are {process_years:?}") })?; @@ -266,21 +266,21 @@ fn validate_secondary_flows( .collect(); // Get the non-primary io flows for all years, if any, arranged by (commodity, region) - let iter = iproduct!(process.years.iter(), process.regions.iter()); + let iter = iproduct!(process.years.clone(), process.regions.iter()); let mut flows: HashMap<(CommodityID, RegionID), Vec<&ProcessFlow>> = HashMap::new(); let mut number_of_years: HashMap<(CommodityID, RegionID), u32> = HashMap::new(); - for (&year, region_id) in iter { - let flow = map[&(region_id.clone(), year)] - .iter() - .filter_map(|(commodity_id, flow)| { + for (year, region_id) in iter { + if let Some(commodity_map) = map.get(&(region_id.clone(), year)) { + let flow = commodity_map.iter().filter_map(|(commodity_id, flow)| { (Some(commodity_id) != process.primary_output.as_ref()) .then_some(((commodity_id.clone(), region_id.clone()), flow)) }); - for (key, value) in flow { - flows.entry(key.clone()).or_default().push(value); - if required_years.contains(&&year) { - *number_of_years.entry(key).or_default() += 1; + for (key, value) in flow { + flows.entry(key.clone()).or_default().push(value); + if required_years.contains(&&year) { + *number_of_years.entry(key).or_default() += 1; + } } } } @@ -338,15 +338,15 @@ mod tests { fn build_maps( process: Process, flows: I, - years: Option<&Vec>, + years: Option>, ) -> (ProcessMap, HashMap) where I: Clone + Iterator, { - let years = years.unwrap_or(&process.years); + let years = years.unwrap_or(process.years.clone().collect()); let map: Rc> = Rc::new(flows.clone().collect()); let flows_inner = iproduct!(&process.regions, years) - .map(|(region_id, year)| ((region_id.clone(), *year), map.clone())) + .map(|(region_id, year)| ((region_id.clone(), year), map.clone())) .collect(); let flows = hash_map! {process.id.clone() => flows_inner}; let processes = iter::once((process.id.clone(), process.into())).collect(); @@ -379,7 +379,7 @@ mod tests { #[from(sed_commodity)] commodity2: Commodity, process: Process, ) { - let milestone_years = vec![2010, 2020]; + let milestone_years: Vec = vec![2010, 2020]; let commodity1 = Rc::new(commodity1); let commodity2 = Rc::new(commodity2); let (mut processes, flows_map) = build_maps( @@ -471,7 +471,7 @@ mod tests { (commodity2.id.clone(), flow(commodity2.clone(), 2.0)), ] .into_iter(), - Some(&flow_years), + Some(flow_years), ); let res = validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years); @@ -497,7 +497,7 @@ mod tests { (commodity2.id.clone(), flow(commodity2.clone(), -2.0)), ] .into_iter(), - Some(&milestone_years), + Some(milestone_years.clone()), ); assert!( validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years) diff --git a/src/input/process/parameter.rs b/src/input/process/parameter.rs index 02a2d23ce..f964445ae 100644 --- a/src/input/process/parameter.rs +++ b/src/input/process/parameter.rs @@ -86,18 +86,18 @@ impl ProcessParameterRaw { pub fn read_process_parameters( model_dir: &Path, processes: &ProcessMap, - base_year: u32, + milestone_years: &[u32], ) -> Result> { let file_path = model_dir.join(PROCESS_PARAMETERS_FILE_NAME); let iter = read_csv::(&file_path)?; - read_process_parameters_from_iter(iter, processes, base_year) + read_process_parameters_from_iter(iter, processes, milestone_years) .with_context(|| input_err_msg(&file_path)) } fn read_process_parameters_from_iter( iter: I, processes: &ProcessMap, - base_year: u32, + milestone_years: &[u32], ) -> Result> where I: Iterator, @@ -110,8 +110,8 @@ where .with_context(|| format!("Process {} not found", param_raw.process_id))?; // Get years - let process_years = &process.years; - let parameter_years = parse_year_str(¶m_raw.commission_years, process_years) + let process_years: Vec = process.years.clone().collect(); + let parameter_years = parse_year_str(¶m_raw.commission_years, &process_years) .with_context(|| { format!("Invalid year for process {id}. Valid years are {process_years:?}") })?; @@ -133,7 +133,7 @@ where } } - check_process_parameters(processes, &map, base_year)?; + check_process_parameters(processes, &map, milestone_years)?; Ok(map) } @@ -142,22 +142,25 @@ where fn check_process_parameters( processes: &ProcessMap, map: &HashMap, - base_year: u32, + milestone_years: &[u32], ) -> Result<()> { for (process_id, process) in processes { let parameters = map .get(process_id) .with_context(|| format!("Missing parameters for process {process_id}"))?; - let reference_years = &process.years; let reference_regions = &process.regions; - // Only give an error for missing parameters >=base_year, so that users are not obliged to - // supply them for every valid year before the time horizon + // Only give an error for missing parameters in milestone years, so that users are not + // obliged to supply them for every valid year before the time horizon let mut missing_keys = Vec::new(); - for year in reference_years.iter().filter(|year| **year >= base_year) { + for year in process + .years + .clone() + .filter(|y| milestone_years.contains(y)) + { for region in reference_regions { - let key = (region.clone(), *year); + let key = (region.clone(), year); if !parameters.contains_key(&key) { missing_keys.push(key); } @@ -232,10 +235,10 @@ mod tests { ) { let mut param_map: HashMap = HashMap::new(); let process_id = processes.keys().next().unwrap().clone(); - let base_year = 2010; + let milestone_years: Vec = vec![2010, 2020]; param_map.insert(process_id, process_parameter_map.clone()); - let result = check_process_parameters(&processes, ¶m_map, base_year); + let result = check_process_parameters(&processes, ¶m_map, &milestone_years); assert!(result.is_ok()); } @@ -247,13 +250,13 @@ mod tests { ) { let mut param_map: HashMap = HashMap::new(); let process_id = processes.keys().next().unwrap().clone(); - let base_year = 2015; + let milestone_years: Vec = vec![2015, 2020]; // Remove one entry before base_year process_parameter_map.remove(&(region_id, 2012)).unwrap(); param_map.insert(process_id, process_parameter_map); - let result = check_process_parameters(&processes, ¶m_map, base_year); + let result = check_process_parameters(&processes, ¶m_map, &milestone_years); assert!(result.is_ok()); } @@ -265,13 +268,13 @@ mod tests { ) { let mut param_map: HashMap = HashMap::new(); let process_id = processes.keys().next().unwrap().clone(); - let base_year = 2010; + let milestone_years: Vec = vec![2010, 2020]; // Remove one region-year key to simulate missing parameter process_parameter_map.remove(&(region_id, 2010)).unwrap(); param_map.insert(process_id, process_parameter_map); - let result = check_process_parameters(&processes, ¶m_map, base_year); + let result = check_process_parameters(&processes, ¶m_map, &milestone_years); assert_error!( result, "Process process1 is missing parameters for the following regions and years: \ diff --git a/src/process.rs b/src/process.rs index 97c183eeb..63af58c98 100644 --- a/src/process.rs +++ b/src/process.rs @@ -42,9 +42,7 @@ pub struct Process { /// A human-readable description for the process (e.g. dry gas extraction) pub description: String, /// The years in which this process is available for investment. - /// - /// These years must be sorted and unique, else it's a logic error. - pub years: Vec, + pub years: RangeInclusive, /// Limits on activity for each time slice (as a fraction of maximum) pub activity_limits: ProcessActivityLimitsMap, /// Maximum annual commodity flows for this process @@ -66,7 +64,7 @@ pub struct Process { impl Process { /// Whether the process can be commissioned in a given year pub fn active_for_year(&self, year: u32) -> bool { - self.years.binary_search(&year).is_ok() + self.years.contains(&year) } } From 53bb216e15fff45773a08566de4e94137c61b935 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 21 Nov 2025 14:21:25 +0000 Subject: [PATCH 2/5] Validate comission year of input assets. --- src/input/asset.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/input/asset.rs b/src/input/asset.rs index 004baa1dd..f4a5f279c 100644 --- a/src/input/asset.rs +++ b/src/input/asset.rs @@ -6,7 +6,7 @@ use crate::id::IDCollection; use crate::process::ProcessMap; use crate::region::RegionID; use crate::units::Capacity; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, ensure}; use indexmap::IndexSet; use itertools::Itertools; use serde::Deserialize; @@ -78,6 +78,31 @@ where .with_context(|| format!("Invalid process ID: {}", &asset.process_id))?; let region_id = region_ids.get_id(&asset.region_id)?; + // Validate commission year. It should be within the process valid range... + ensure!( + process.years.contains(&asset.commission_year), + "Agent {} has asset with commission year {}, not within process {} commission years: {:?}", + asset.agent_id, + asset.commission_year, + asset.process_id, + process.years + ); + // ... and also have associated process parameters and flows + ensure!( + process.parameters.contains_key(&(region_id.clone(), asset.commission_year)), + "Parameters for process {} do not contain entry for year {}, required for asset in agent {}", + asset.process_id, + asset.commission_year, + asset.agent_id, + ); + ensure!( + process.flows.contains_key(&(region_id.clone(), asset.commission_year)), + "Flows for process {} do not contain entry for year {}, required for asset in agent {}", + asset.process_id, + asset.commission_year, + asset.agent_id, + ); + Asset::new_future_with_max_decommission( agent_id.clone(), Rc::clone(process), From 9929945cedcf21e03573d034a020b0faa024b69e Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 24 Nov 2025 09:10:01 +0000 Subject: [PATCH 3/5] Use milestone years to validate availability --- src/input/process.rs | 3 +-- src/input/process/availability.rs | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/input/process.rs b/src/input/process.rs index 9415a5930..15f96bc1a 100644 --- a/src/input/process.rs +++ b/src/input/process.rs @@ -56,10 +56,9 @@ pub fn read_processes( time_slice_info: &TimeSliceInfo, milestone_years: &[u32], ) -> Result { - let base_year = milestone_years[0]; let mut processes = read_processes_file(model_dir, milestone_years, region_ids, commodities)?; let mut activity_limits = - read_process_availabilities(model_dir, &processes, time_slice_info, base_year)?; + read_process_availabilities(model_dir, &processes, time_slice_info, milestone_years)?; let mut flows = read_process_flows(model_dir, &mut processes, commodities, milestone_years)?; let mut parameters = read_process_parameters(model_dir, &processes, milestone_years)?; diff --git a/src/input/process/availability.rs b/src/input/process/availability.rs index 87292ac53..b35c2e364 100644 --- a/src/input/process/availability.rs +++ b/src/input/process/availability.rs @@ -75,7 +75,7 @@ enum LimitType { /// * `model_dir` - Folder containing model configuration files /// * `processes` - Map of processes /// * `time_slice_info` - Information about seasons and times of day -/// * `base_year` - First milestone year of simulation +/// * `milestone_years` - Milestone years of simulation /// /// # Returns /// @@ -85,7 +85,7 @@ pub fn read_process_availabilities( model_dir: &Path, processes: &ProcessMap, time_slice_info: &TimeSliceInfo, - base_year: u32, + milestone_years: &[u32], ) -> Result> { let file_path = model_dir.join(PROCESS_AVAILABILITIES_FILE_NAME); let process_availabilities_csv = read_csv(&file_path)?; @@ -93,7 +93,7 @@ pub fn read_process_availabilities( process_availabilities_csv, processes, time_slice_info, - base_year, + milestone_years, ) .with_context(|| input_err_msg(&file_path)) } @@ -105,7 +105,7 @@ pub fn read_process_availabilities( /// * `iter` - Iterator of raw process availability records /// * `processes` - Map of processes /// * `time_slice_info` - Information about seasons and times of day -/// * `base_year` - First milestone year of simulation +/// * `milestone_years` - Milestone years of simulation /// /// # Returns /// @@ -115,7 +115,7 @@ fn read_process_availabilities_from_iter( iter: I, processes: &ProcessMap, time_slice_info: &TimeSliceInfo, - base_year: u32, + milestone_years: &[u32], ) -> Result> where I: Iterator, @@ -162,7 +162,7 @@ where } } - validate_activity_limits_maps(&map, processes, time_slice_info, base_year)?; + validate_activity_limits_maps(&map, processes, time_slice_info, milestone_years)?; Ok(map) } @@ -172,7 +172,7 @@ fn validate_activity_limits_maps( all_availabilities: &HashMap, processes: &ProcessMap, time_slice_info: &TimeSliceInfo, - base_year: u32, + milestone_years: &[u32], ) -> Result<()> { for (process_id, process) in processes { // A map of maps: the outer map is keyed by region and year; the inner one by time slice @@ -180,7 +180,7 @@ fn validate_activity_limits_maps( .get(process_id) .with_context(|| format!("Missing availabilities for process {process_id}"))?; - check_missing_milestone_years(process, map_for_process, base_year)?; + check_missing_milestone_years(process, map_for_process, milestone_years)?; check_missing_time_slices(process, map_for_process, time_slice_info)?; } @@ -195,11 +195,10 @@ fn validate_activity_limits_maps( fn check_missing_milestone_years( process: &Process, map_for_process: &ProcessActivityLimitsMap, - base_year: u32, + milestone_years: &[u32], ) -> Result<()> { - let process_milestone_years = process.years.clone().filter(|&year| year >= base_year); let mut missing = Vec::new(); - for (region_id, year) in iproduct!(&process.regions, process_milestone_years) { + for (region_id, &year) in iproduct!(&process.regions, milestone_years) { if !map_for_process.contains_key(&(region_id.clone(), year)) { missing.push((region_id, year)); } From 0ca440ba34d53d67422ae8257981d3b9308018bc Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 24 Nov 2025 09:16:08 +0000 Subject: [PATCH 4/5] Warn when using default years for process --- src/input/process.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/input/process.rs b/src/input/process.rs index 15f96bc1a..61caf993e 100644 --- a/src/input/process.rs +++ b/src/input/process.rs @@ -10,6 +10,7 @@ use crate::time_slice::TimeSliceInfo; use crate::units::ActivityPerCapacity; use anyhow::{Context, Ok, Result, ensure}; use indexmap::IndexSet; +use log::warn; use serde::Deserialize; use std::path::Path; use std::rc::Rc; @@ -99,10 +100,21 @@ where { let mut processes = ProcessMap::new(); for process_raw in iter { - let start_year = process_raw.start_year.unwrap_or(milestone_years[0]); - let end_year = process_raw - .end_year - .unwrap_or(*milestone_years.last().unwrap()); + let start_year = process_raw.start_year.unwrap_or_else(|| { + warn!( + "Using default start year {} for process {}.", + milestone_years[0], process_raw.id + ); + milestone_years[0] + }); + let end_year = process_raw.end_year.unwrap_or_else(|| { + warn!( + "Using default end year {} for process {}.", + milestone_years.last().unwrap(), + process_raw.id + ); + *milestone_years.last().unwrap() + }); // Check year range is valid ensure!( @@ -110,9 +122,6 @@ where "Error in parameter for process {}: start_year > end_year", process_raw.id ); - - // Select process years. It is possible for assets to have been commissioned before the - // simulation's time horizon, so assume that all years >=start_year are valid too. let years = start_year..=end_year; // Parse region ID From 55fe6aa7eace7c9c25b8fa80072b8bfcc6e04c50 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Mon, 24 Nov 2025 11:37:09 +0000 Subject: [PATCH 5/5] Fix years for availabilities --- src/input/process/availability.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/input/process/availability.rs b/src/input/process/availability.rs index b35c2e364..4fa68eb32 100644 --- a/src/input/process/availability.rs +++ b/src/input/process/availability.rs @@ -197,8 +197,12 @@ fn check_missing_milestone_years( map_for_process: &ProcessActivityLimitsMap, milestone_years: &[u32], ) -> Result<()> { + let process_milestone_years = process + .years + .clone() + .filter(|year| milestone_years.contains(year)); let mut missing = Vec::new(); - for (region_id, &year) in iproduct!(&process.regions, milestone_years) { + for (region_id, year) in iproduct!(&process.regions, process_milestone_years) { if !map_for_process.contains_key(&(region_id.clone(), year)) { missing.push((region_id, year)); }