Check flows in milestone years only

src/asset.rs

@@ -1023,8 +1023,8 @@ mod tests {
         let agent_id = AgentID("agent1".into());
         let region_id = RegionID("GBR".into());
         assert_error!(
-            Asset::new_future(agent_id, process.into(), region_id, Capacity(1.0), 2009),
-            "Process process1 does not operate in the year 2009"
+            Asset::new_future(agent_id, process.into(), region_id, Capacity(1.0), 2007),
+            "Process process1 does not operate in the year 2007"
         );
     }
 

src/fixture.rs

@@ -148,13 +148,19 @@ pub fn process(
     region_ids: IndexSet<RegionID>,
     process_parameter_map: ProcessParameterMap,
 ) -> Process {
-    let years = vec![2010, 2015, 2020];
+    let milestone_years = vec![2010, 2015, 2020];
+    // The process start year is before the base year
+    let years = vec![2008, 2009]
+        .iter()
+        .chain(&milestone_years)
+        .cloned()
+        .collect();
 
     // Create maps with (empty) entries for every region/year combo
-    let activity_limits = iproduct!(region_ids.iter(), years.iter())
+    let activity_limits = iproduct!(region_ids.iter(), milestone_years.iter())
         .map(|(region_id, year)| ((region_id.clone(), *year), Rc::new(HashMap::new())))
         .collect();
-    let flows = iproduct!(region_ids.iter(), years.iter())
+    let flows = iproduct!(region_ids.iter(), milestone_years.iter())
         .map(|(region_id, year)| ((region_id.clone(), *year), Rc::new(IndexMap::new())))
         .collect();
     Process {

src/input/process.rs

@@ -61,7 +61,7 @@ pub fn read_processes(
     let mut processes = read_processes_file(model_dir, milestone_years, region_ids, commodities)?;
     let mut activity_limits =
         read_process_availabilities(model_dir, &processes, time_slice_info, base_year)?;
-    let mut flows = read_process_flows(model_dir, &mut processes, commodities)?;
+    let mut flows = read_process_flows(model_dir, &mut processes, commodities, milestone_years)?;
     let mut parameters = read_process_parameters(model_dir, &processes, base_year)?;
 
     // Add data to Process objects

src/input/process/flow.rs

@@ -62,10 +62,11 @@ pub fn read_process_flows(
     model_dir: &Path,
     processes: &mut ProcessMap,
     commodities: &CommodityMap,
+    milestone_years: &[u32],
 ) -> Result<HashMap<ProcessID, ProcessFlowsMap>> {
     let file_path = model_dir.join(PROCESS_FLOWS_FILE_NAME);
     let process_flow_csv = read_csv(&file_path)?;
-    read_process_flows_from_iter(process_flow_csv, processes, commodities)
+    read_process_flows_from_iter(process_flow_csv, processes, commodities, milestone_years)
         .with_context(|| input_err_msg(&file_path))
 }
 
@@ -74,6 +75,7 @@ fn read_process_flows_from_iter<I>(
     iter: I,
     processes: &mut ProcessMap,
     commodities: &CommodityMap,
+    milestone_years: &[u32],
 ) -> Result<HashMap<ProcessID, ProcessFlowsMap>>
 where
     I: Iterator<Item = ProcessFlowRaw>,
@@ -134,38 +136,46 @@ where
         }
     }
 
-    validate_flows_and_update_primary_output(processes, &flows_map)?;
-    validate_secondary_flows(processes, &flows_map)?;
+    validate_flows_and_update_primary_output(processes, &flows_map, milestone_years)?;
+    validate_secondary_flows(processes, &flows_map, milestone_years)?;
 
     Ok(flows_map)
 }
 
 fn validate_flows_and_update_primary_output(
     processes: &mut ProcessMap,
     flows_map: &HashMap<ProcessID, ProcessFlowsMap>,
+    milestone_years: &[u32],
 ) -> Result<()> {
     for (process_id, process) in processes.iter_mut() {
         let map = flows_map
             .get(process_id)
             .with_context(|| format!("Missing flows map for process {process_id}"))?;
 
+        // Flows are required for all milestone years within the process years of activity
+        let required_years = milestone_years
+            .iter()
+            .filter(|&y| process.years.contains(y));
+        let region_year: Vec<(&RegionID, &u32)> =
+            iproduct!(process.regions.iter(), required_years).collect();
+
         ensure!(
-            map.len() == process.years.len() * process.regions.len(),
-            "Flows map for process {process_id} does not cover all regions and years"
+            region_year
+                .iter()
+                .all(|(region_id, year)| map.contains_key(&((*region_id).clone(), **year))),
+            "Flows map for process {process_id} does not cover all regions and required years"
         );
 
-        let mut iter = iproduct!(process.years.iter(), process.regions.iter());
-
         let primary_output = if let Some(primary_output) = &process.primary_output {
             Some(primary_output.clone())
         } else {
-            let (year, region_id) = iter.next().unwrap();
+            let (region_id, year) = region_year[0];
             infer_primary_output(&map[&(region_id.clone(), *year)]).with_context(|| {
                 format!("Could not infer primary_output for process {process_id}")
             })?
         };
 
-        for (&year, region_id) in iter {
+        for (region_id, &year) in region_year {
             let flows = &map[&(region_id.clone(), year)];
 
             // Check that the process has flows for this region/year
@@ -236,21 +246,29 @@ fn check_flows_primary_output(
     Ok(())
 }
 
-/// Checks that non-primary io are defined for all years (within a region) and that
-/// they are only inputs or only outputs in all years.
+/// Checks that non-primary io are defined for all milestone years, at least, (within a region) and
+/// that they are only inputs or only outputs in all years.
 fn validate_secondary_flows(
     processes: &mut ProcessMap,
     flows_map: &HashMap<ProcessID, ProcessFlowsMap>,
+    milestone_years: &[u32],
 ) -> Result<()> {
     for (process_id, process) in processes.iter() {
         // Get the flows for this process - there should be no error, as was checked already
         let map = flows_map
             .get(process_id)
             .with_context(|| format!("Missing flows map for process {process_id}"))?;
 
+        // Flows are required for all milestone years within the process years of activity
+        let required_years: Vec<&u32> = milestone_years
+            .iter()
+            .filter(|&y| process.years.contains(y))
+            .collect();
+
         // Get the non-primary io flows for all years, if any, arranged by (commodity, region)
         let iter = iproduct!(process.years.iter(), process.regions.iter());
         let mut flows: HashMap<(CommodityID, RegionID), Vec<&ProcessFlow>> = HashMap::new();
+        let mut number_of_years: HashMap<(CommodityID, RegionID), u32> = HashMap::new();
         for (&year, region_id) in iter {
             let flow = map[&(region_id.clone(), year)]
                 .iter()
@@ -260,17 +278,22 @@ fn validate_secondary_flows(
                 });
 
             for (key, value) in flow {
-                flows.entry(key).or_default().push(value);
+                flows.entry(key.clone()).or_default().push(value);
+                if required_years.contains(&&year) {
+                    *number_of_years.entry(key).or_default() += 1;
+                }
             }
         }
 
         // Finally we check that the flows for a given commodity and region are defined for all
-        // years and that they are all inputs or all outputs
+        // milestone years and that they are all inputs or all outputs. This later check is done
+        // for all years in the process range, required or not.
         for ((commodity_id, region_id), value) in &flows {
             ensure!(
-                value.len() == process.years.len(),
+                number_of_years[&(commodity_id.clone(), region_id.clone())]
+                    == required_years.len().try_into().unwrap(),
                 "Flow of commodity {commodity_id} in region {region_id} for process {process_id} \
-                does not cover all years"
+                does not cover all milestone years within the process range of activity."
             );
             let input_or_zero = value
                 .iter()
@@ -315,12 +338,14 @@ mod tests {
     fn build_maps<I>(
         process: Process,
         flows: I,
+        years: Option<&Vec<u32>>,
     ) -> (ProcessMap, HashMap<ProcessID, ProcessFlowsMap>)
     where
         I: Clone + Iterator<Item = (CommodityID, ProcessFlow)>,
     {
+        let years = years.unwrap_or(&process.years);
         let map: Rc<IndexMap<_, _>> = Rc::new(flows.clone().collect());
-        let flows_inner = iproduct!(&process.regions, &process.years)
+        let flows_inner = iproduct!(&process.regions, years)
             .map(|(region_id, year)| ((region_id.clone(), *year), map.clone()))
             .collect();
         let flows = hash_map! {process.id.clone() => flows_inner};
@@ -331,12 +356,17 @@ mod tests {
 
     #[rstest]
     fn single_output_infer_primary(#[from(svd_commodity)] commodity: Commodity, process: Process) {
+        let milestone_years = vec![2010, 2020];
         let commodity = Rc::new(commodity);
         let (mut processes, flows_map) = build_maps(
             process,
             std::iter::once((commodity.id.clone(), flow(commodity.clone(), 1.0))),
+            None,
+        );
+        assert!(
+            validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years)
+                .is_ok()
         );
-        assert!(validate_flows_and_update_primary_output(&mut processes, &flows_map).is_ok());
         assert_eq!(
             processes.values().exactly_one().unwrap().primary_output,
             Some(commodity.id.clone())
@@ -349,6 +379,7 @@ mod tests {
         #[from(sed_commodity)] commodity2: Commodity,
         process: Process,
     ) {
+        let milestone_years = vec![2010, 2020];
         let commodity1 = Rc::new(commodity1);
         let commodity2 = Rc::new(commodity2);
         let (mut processes, flows_map) = build_maps(
@@ -358,8 +389,10 @@ mod tests {
                 (commodity2.id.clone(), flow(commodity2.clone(), 2.0)),
             ]
             .into_iter(),
+            None,
         );
-        let res = validate_flows_and_update_primary_output(&mut processes, &flows_map);
+        let res =
+            validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years);
         assert_error!(res, "Could not infer primary_output for process process1");
     }
 
@@ -369,6 +402,7 @@ mod tests {
         #[from(sed_commodity)] commodity2: Commodity,
         process: Process,
     ) {
+        let milestone_years = vec![2010, 2020];
         let commodity1 = Rc::new(commodity1);
         let commodity2 = Rc::new(commodity2);
         let mut process = process;
@@ -380,8 +414,12 @@ mod tests {
                 (commodity2.id.clone(), flow(commodity2.clone(), 2.0)),
             ]
             .into_iter(),
+            None,
+        );
+        assert!(
+            validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years)
+                .is_ok()
         );
-        assert!(validate_flows_and_update_primary_output(&mut processes, &flows_map).is_ok());
         assert_eq!(
             processes.values().exactly_one().unwrap().primary_output,
             Some(commodity2.id.clone())
@@ -394,6 +432,7 @@ mod tests {
         #[from(sed_commodity)] commodity2: Commodity,
         process: Process,
     ) {
+        let milestone_years = vec![2010, 2020];
         let commodity1 = Rc::new(commodity1);
         let commodity2 = Rc::new(commodity2);
         let (mut processes, flows_map) = build_maps(
@@ -403,11 +442,66 @@ mod tests {
                 (commodity2.id.clone(), flow(commodity2.clone(), -2.0)),
             ]
             .into_iter(),
+            None,
+        );
+        assert!(
+            validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years)
+                .is_ok()
         );
-        assert!(validate_flows_and_update_primary_output(&mut processes, &flows_map).is_ok());
         assert_eq!(
             processes.values().exactly_one().unwrap().primary_output,
             None
         );
     }
+
+    #[rstest]
+    fn flows_not_in_all_milestone_years(
+        #[from(svd_commodity)] commodity1: Commodity,
+        #[from(sed_commodity)] commodity2: Commodity,
+        process: Process,
+    ) {
+        let milestone_years = vec![2010, 2015, 2020];
+        let flow_years = vec![2010, 2020];
+        let commodity1 = Rc::new(commodity1);
+        let commodity2 = Rc::new(commodity2);
+        let (mut processes, flows_map) = build_maps(
+            process,
+            [
+                (commodity1.id.clone(), flow(commodity1.clone(), 1.0)),
+                (commodity2.id.clone(), flow(commodity2.clone(), 2.0)),
+            ]
+            .into_iter(),
+            Some(&flow_years),
+        );
+        let res =
+            validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years);
+        assert_error!(
+            res,
+            "Flows map for process process1 does not cover all regions and required years"
+        );
+    }
+
+    #[rstest]
+    fn flows_only_milestone_years(
+        #[from(svd_commodity)] commodity1: Commodity,
+        #[from(sed_commodity)] commodity2: Commodity,
+        process: Process,
+    ) {
+        let milestone_years = vec![2010, 2015, 2020];
+        let commodity1 = Rc::new(commodity1);
+        let commodity2 = Rc::new(commodity2);
+        let (mut processes, flows_map) = build_maps(
+            process,
+            [
+                (commodity1.id.clone(), flow(commodity1.clone(), 1.0)),
+                (commodity2.id.clone(), flow(commodity2.clone(), -2.0)),
+            ]
+            .into_iter(),
+            Some(&milestone_years),
+        );
+        assert!(
+            validate_flows_and_update_primary_output(&mut processes, &flows_map, &milestone_years)
+                .is_ok()
+        );
+    }
 }