Skip to content

Create ProcessFlowRaw struct #198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TinyMarsh merged 10 commits into from
Dec 20, 2024
Merged

Create ProcessFlowRaw struct #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
deec073
Create ProcessFlowRaw struct
TinyMarsh Nov 11, 2024
4c3b3a3
Merge branch 'main' into flow-commodity-field
TinyMarsh Dec 9, 2024
e30052a
Convert method for ProcessFlow
TinyMarsh Dec 11, 2024
31788ee
Correctly fetch Commodity and convert to ProcessFlow
TinyMarsh Dec 13, 2024
f1bab95
move into_flow logic in to read_process_flows_from_iter
TinyMarsh Dec 16, 2024
d1bdc48
Add tests
TinyMarsh Dec 16, 2024
c4fdd9f
Consistent passing of model_dir
TinyMarsh Dec 16, 2024
5f7bcf3
Refactor and move processing logic into read_process_flows_from_iter.…
TinyMarsh Dec 19, 2024
ecd14c2
Merge branch 'main' into flow-commodity-field
TinyMarsh Dec 19, 2024
c29a99a
Addresses PR feedback
TinyMarsh Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 0 additions & 82 deletions src/input.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,28 +166,6 @@ where
}
}

/// Read a CSV file, grouping the entries by ID
///
/// # Arguments
///
/// * `file_path` - Path to CSV file
/// * `ids` - All possible IDs that will be encountered
///
/// # Returns
///
/// A HashMap with ID as a key and a vector of CSV data as a value or an error.
pub fn read_csv_grouped_by_id<T>(
file_path: &Path,
ids: &HashSet<Rc<str>>,
) -> Result<HashMap<Rc<str>, Vec<T>>>
where
T: HasID + DeserializeOwned,
{
read_csv(file_path)?
.into_id_map(ids)
.with_context(|| input_err_msg(file_path))
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down Expand Up @@ -285,64 +263,4 @@ mod tests {
assert!(deserialise_f64(f64::NAN).is_err());
assert!(deserialise_f64(f64::INFINITY).is_err());
}

fn create_ids() -> HashSet<Rc<str>> {
HashSet::from(["A".into(), "B".into()])
}

#[test]
fn test_read_csv_grouped_by_id() {
let dir = tempdir().unwrap();
let file_path = dir.path().join("data.csv");
{
let file_path: &Path = &file_path; // cast
let mut file = File::create(file_path).unwrap();
writeln!(file, "id,value\nA,1\nB,2\nA,3").unwrap();
}

let expected = HashMap::from([
(
"A".into(),
vec![
Record {
id: "A".to_string(),
value: 1,
},
Record {
id: "A".to_string(),
value: 3,
},
],
),
(
"B".into(),
vec![Record {
id: "B".to_string(),
value: 2,
}],
),
]);
let process_ids = create_ids();
let file_path = dir.path().join("data.csv");
let map = read_csv_grouped_by_id::<Record>(&file_path, &process_ids);
assert_eq!(expected, map.unwrap());
}

#[test]
#[should_panic]
fn test_read_csv_grouped_by_id_duplicate() {
let dir = tempdir().unwrap();
let file_path = dir.path().join("data.csv");
{
let file_path: &Path = &file_path; // cast
let mut file = File::create(file_path).unwrap();

// NB: Process ID "C" isn't valid
writeln!(file, "process_id,value\nA,1\nB,2\nC,3").unwrap();
}

// Check that it fails if a non-existent process ID is provided
let process_ids = create_ids();
read_csv_grouped_by_id::<Record>(&file_path, &process_ids).unwrap();
}
}
192 changes: 180 additions & 12 deletions src/process.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,21 +67,33 @@ pub enum FlowType {
Flexible,
}

#[derive(PartialEq, Debug, Deserialize)]
struct ProcessFlowRaw {
process_id: String,
commodity_id: String,
flow: f64,
#[serde(default)]
flow_type: FlowType,
#[serde(deserialize_with = "deserialise_flow_cost")]
flow_cost: f64,
}

define_process_id_getter! {ProcessFlowRaw}

#[derive(PartialEq, Debug, Deserialize, Clone)]
pub struct ProcessFlow {
/// A unique identifier for the process (typically uses a structured naming convention).
pub process_id: String,
/// Identifies the commodity for the specified flow
pub commodity_id: String,
pub commodity: Rc<Commodity>,
/// Commodity flow quantity relative to other commodity flows. +ve value indicates flow out, -ve value indicates flow in.
pub flow: f64,
#[serde(default)]
/// Identifies if a flow is fixed or flexible.
pub flow_type: FlowType,
#[serde(deserialize_with = "deserialise_flow_cost")]
/// Cost per unit flow. For example, cost per unit of natural gas produced. Differs from var_opex because the user can apply it to any specified flow, whereas var_opex applies to pac flow.
pub flow_cost: f64,
}

define_process_id_getter! {ProcessFlow}

/// Custom deserialiser for flow cost - treat empty fields as 0.0
Expand Down Expand Up @@ -278,6 +290,42 @@ fn read_process_availabilities(
.with_context(|| input_err_msg(&file_path))
}

/// Read 'ProcessFlowRaw' records from an iterator and convert them into 'ProcessFlow' records.
fn read_process_flows_from_iter<I>(
iter: I,
process_ids: &HashSet<Rc<str>>,
commodities: &HashMap<Rc<str>, Rc<Commodity>>,
) -> Result<HashMap<Rc<str>, Vec<ProcessFlow>>>
where
I: Iterator<Item = ProcessFlowRaw>,
{
iter.map(|flow_raw| -> Result<ProcessFlow> {
let commodity = commodities
.get(flow_raw.commodity_id.as_str())
.with_context(|| format!("{} is not a valid commodity ID", &flow_raw.commodity_id))?;

Ok(ProcessFlow {
process_id: flow_raw.process_id,
commodity: Rc::clone(commodity),
flow: flow_raw.flow,
flow_type: flow_raw.flow_type,
flow_cost: flow_raw.flow_cost,
})
})
.process_results(|iter| iter.into_id_map(process_ids))?
}

fn read_process_flows(
model_dir: &Path,
process_ids: &HashSet<Rc<str>>,
commodities: &HashMap<Rc<str>, Rc<Commodity>>,
) -> Result<HashMap<Rc<str>, Vec<ProcessFlow>>> {
let file_path = model_dir.join(PROCESS_FLOWS_FILE_NAME);
let process_flow_csv = read_csv(&file_path)?;
read_process_flows_from_iter(process_flow_csv, process_ids, commodities)
.with_context(|| input_err_msg(&file_path))
}

fn read_process_parameters_from_iter<I>(
iter: I,
process_ids: &HashSet<Rc<str>>,
Expand All @@ -290,18 +338,15 @@ where
for param in iter {
let param = param.into_parameter(year_range)?;
let id = process_ids.get_id(&param.process_id)?;

ensure!(
params.insert(Rc::clone(&id), param).is_none(),
"More than one parameter provided for process {id}"
);
}

ensure!(
params.len() == process_ids.len(),
"Each process must have an associated parameter"
);

Ok(params)
}

Expand Down Expand Up @@ -383,7 +428,7 @@ fn validate_pac_flows(
// Find the flow associated with the PAC
let flow = flows
.iter()
.find(|item| *item.commodity_id.as_str() == *pac.id)
.find(|item| *item.commodity.id == *pac.id)
.with_context(|| {
format!(
"PAC {} for process {} must have an associated flow",
Expand Down Expand Up @@ -450,8 +495,7 @@ pub fn read_processes(
let process_ids = HashSet::from_iter(descriptions.keys().cloned());

let mut availabilities = read_process_availabilities(model_dir, &process_ids, time_slice_info)?;
let file_path = model_dir.join(PROCESS_FLOWS_FILE_NAME);
let mut flows = read_csv_grouped_by_id(&file_path, &process_ids)?;
let mut flows = read_process_flows(model_dir, &process_ids, commodities)?;
let mut pacs = read_process_pacs(model_dir, &process_ids, commodities, &flows)?;
let mut parameters = read_process_parameters(model_dir, &process_ids, year_range)?;
let file_path = model_dir.join(PROCESS_REGIONS_FILE_NAME);
Expand Down Expand Up @@ -485,6 +529,7 @@ pub fn read_processes(

#[cfg(test)]
mod tests {

use crate::commodity::{CommodityCostMap, CommodityType};
use crate::time_slice::TimeSliceLevel;

Expand Down Expand Up @@ -629,6 +674,129 @@ mod tests {
);
}

#[test]
fn test_read_process_flows_from_iter_good() {
let process_ids = ["id1".into(), "id2".into()].into_iter().collect();
let commodities: HashMap<Rc<str>, Rc<Commodity>> = ["commodity1", "commodity2"]
.into_iter()
.map(|id| {
let commodity = Commodity {
id: id.into(),
description: "Some description".into(),
kind: CommodityType::InputCommodity,
time_slice_level: TimeSliceLevel::Annual,
costs: CommodityCostMap::new(),
demand_by_region: HashMap::new(),
};

(Rc::clone(&commodity.id), commodity.into())
})
.collect();

let flows_raw = [
ProcessFlowRaw {
process_id: "id1".into(),
commodity_id: "commodity1".into(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
},
ProcessFlowRaw {
process_id: "id1".into(),
commodity_id: "commodity2".into(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
},
ProcessFlowRaw {
process_id: "id2".into(),
commodity_id: "commodity1".into(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
},
];

let expected = HashMap::from([
(
"id1".into(),
vec![
ProcessFlow {
process_id: "id1".into(),
commodity: commodities.get("commodity1").unwrap().clone(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
},
ProcessFlow {
process_id: "id1".into(),
commodity: commodities.get("commodity2").unwrap().clone(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
},
],
),
(
"id2".into(),
vec![ProcessFlow {
process_id: "id2".into(),
commodity: commodities.get("commodity1").unwrap().clone(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
}],
),
]);

let actual =
read_process_flows_from_iter(flows_raw.into_iter(), &process_ids, &commodities)
.unwrap();
assert_eq!(expected, actual);
}

#[test]
fn test_read_process_flows_from_iter_bad_commodity_id() {
let process_ids = ["id1".into(), "id2".into()].into_iter().collect();
let commodities = ["commodity1", "commodity2"]
.into_iter()
.map(|id| {
let commodity = Commodity {
id: id.into(),
description: "Some description".into(),
kind: CommodityType::InputCommodity,
time_slice_level: TimeSliceLevel::Annual,
costs: CommodityCostMap::new(),
demand_by_region: HashMap::new(),
};

(Rc::clone(&commodity.id), commodity.into())
})
.collect();

let flows_raw = [
ProcessFlowRaw {
process_id: "id1".into(),
commodity_id: "commodity1".into(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
},
ProcessFlowRaw {
process_id: "id1".into(),
commodity_id: "commodity3".into(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
},
];

assert!(
read_process_flows_from_iter(flows_raw.into_iter(), &process_ids, &commodities)
.is_err()
);
}

#[test]
fn test_read_process_parameters_from_iter_good() {
let year_range = 2000..=2100;
Expand Down Expand Up @@ -773,7 +941,7 @@ mod tests {
fn test_read_process_pacs_from_iter() {
// Prepare test data
let process_ids = ["id1".into(), "id2".into()].into_iter().collect();
let commodities = ["commodity1", "commodity2"]
let commodities: HashMap<Rc<str>, Rc<Commodity>> = ["commodity1", "commodity2"]
.into_iter()
.map(|id| {
let commodity = Commodity {
Expand All @@ -796,7 +964,7 @@ mod tests {
.into_iter()
.map(|commodity_id| ProcessFlow {
process_id: process_id.into(),
commodity_id: commodity_id.into(),
commodity: commodities.get(commodity_id).unwrap().clone(),
flow: 1.0,
flow_type: FlowType::Fixed,
flow_cost: 1.0,
Expand Down Expand Up @@ -884,7 +1052,7 @@ mod tests {
.get_mut(&Rc::from("id1"))
.unwrap()
.iter_mut()
.find(|flow| flow.commodity_id == "commodity1")
.find(|flow| flow.commodity.id == "commodity1".into())
.unwrap()
.flow = -1.0;
assert!(
Expand Down