From 0885d0586f477efc75cc436ddfda5fc8a0b38a0f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 7 Jan 2025 12:40:17 +0700 Subject: [PATCH 01/30] aggregate sum tree types --- .../estimated_costs/average_case_costs.rs | 4 +- .../batch/estimated_costs/worst_case_costs.rs | 4 +- grovedb/src/batch/mod.rs | 8 +- grovedb/src/lib.rs | 6 +- grovedb/src/tests/mod.rs | 2 +- merk/src/error.rs | 3 + merk/src/lib.rs | 2 +- merk/src/merk/mod.rs | 33 +-- merk/src/merk/restore.rs | 6 +- merk/src/proofs/tree.rs | 31 ++- merk/src/tree/encoding.rs | 4 +- merk/src/tree/link.rs | 212 +++++++++++++----- merk/src/tree/mod.rs | 110 +++++++-- merk/src/tree/tree_feature_type.rs | 60 ++++- merk/src/tree/walk/mod.rs | 2 +- 15 files changed, 359 insertions(+), 128 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index ef64b0f4d..051025069 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -10,7 +10,7 @@ use std::{ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::RootHashKeyAndSum; +use grovedb_merk::RootHashKeyAndAggregateData; #[cfg(feature = "full")] use grovedb_merk::{ estimated_costs::average_case_costs::{average_case_merk_propagate, EstimatedLayerInformation}, @@ -192,7 +192,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { _flags_update: &mut G, _split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { let mut cost = OperationCost::default(); let layer_element_estimates = cost_return_on_error_no_add!( diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 9bf9a808b..2d25ae81d 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -14,7 +14,7 @@ use grovedb_costs::{ use grovedb_merk::estimated_costs::worst_case_costs::{ worst_case_merk_propagate, WorstCaseLayerInformation, }; -use grovedb_merk::RootHashKeyAndSum; +use grovedb_merk::RootHashKeyAndAggregateData; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; @@ -188,7 +188,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { _flags_update: &mut G, _split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { let mut cost = OperationCost::default(); let worst_case_layer_element_estimates = cost_return_on_error_no_add!( diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 2767ab2b1..13525edaf 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -52,7 +52,7 @@ use grovedb_merk::{ kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, value_hash, NULL_HASH, }, - CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndSum, + CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndAggregateData, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; use grovedb_path::SubtreePath; @@ -695,7 +695,7 @@ trait TreeCache { flags_update: &mut G, split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult; + ) -> CostResult; fn update_base_merk_root_key( &mut self, @@ -1277,7 +1277,7 @@ where flags_update: &mut G, split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { let mut cost = OperationCost::default(); // todo: fix this let p = path.to_path(); @@ -1690,7 +1690,7 @@ where .map_err(|e| Error::CorruptedData(e.to_string())) ); let r = merk - .root_hash_key_and_sum() + .root_hash_key_and_aggregate_data() .add_cost(cost) .map_err(Error::MerkError); diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index f3b2dcc4e..dce0e1b48 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -653,7 +653,7 @@ impl GroveDb { ); let (root_hash, root_key, sum) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_sum().map_err(Error::MerkError) + child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -707,7 +707,7 @@ impl GroveDb { ); let (root_hash, root_key, sum) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_sum().map_err(Error::MerkError) + child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -758,7 +758,7 @@ impl GroveDb { ); let (root_hash, root_key, sum) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_sum().map_err(Error::MerkError) + child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 412636691..d5ec111ea 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -3220,7 +3220,7 @@ mod tests { .unwrap() .expect("expected to get root merk"); let (_, root_key, _) = root_merk - .root_hash_key_and_sum() + .root_hash_key_and_aggregate_data() .unwrap() .expect("expected to get root hash, key and sum"); assert!(root_key.is_some()) diff --git a/merk/src/error.rs b/merk/src/error.rs index c365b898e..f9ba79c0c 100644 --- a/merk/src/error.rs +++ b/merk/src/error.rs @@ -117,6 +117,9 @@ pub enum Error { #[error(transparent)] /// Version error VersionError(grovedb_version::error::GroveVersionError), + + #[error("big sum tree under normal sum tree error {0}")] + BigSumTreeUnderNormalSumTree(String), } impl From for Error { diff --git a/merk/src/lib.rs b/merk/src/lib.rs index d746a885a..cd74d7acd 100644 --- a/merk/src/lib.rs +++ b/merk/src/lib.rs @@ -84,7 +84,7 @@ pub use tree::{CryptoHash, TreeFeatureType}; pub use crate::merk::{ defaults::ROOT_KEY_KEY, prove::{ProofConstructionResult, ProofWithoutEncodingResult}, - IsSumTree, KVIterator, Merk, MerkType, RootHashKeyAndSum, + IsSumTree, KVIterator, Merk, MerkType, RootHashKeyAndAggregateData, }; #[cfg(feature = "full")] pub use crate::visualize::VisualizeableMerk; diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index ee0deccc3..b7b181caa 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -76,6 +76,7 @@ use crate::{ Link, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, }; +use crate::tree::AggregateData; /// Key update types pub struct KeyUpdates { @@ -114,7 +115,7 @@ pub type BatchValue = ( pub type IsSumTree = bool; /// Root hash key and sum -pub type RootHashKeyAndSum = (CryptoHash, Option>, Option); +pub type RootHashKeyAndAggregateData = (CryptoHash, Option>, AggregateData); /// KVIterator allows you to lazily iterate over each kv pair of a subtree pub struct KVIterator<'a, I: RawIterator> { @@ -295,11 +296,11 @@ where res } - /// Returns the total sum value in the Merk tree - pub fn sum(&self) -> Result, Error> { + /// Returns the total aggregate data in the Merk tree + pub fn aggregate_data(&self) -> Result { self.use_tree(|tree| match tree { - None => Ok(None), - Some(tree) => tree.sum(), + None => Ok(AggregateData::NoAggregateData), + Some(tree) => tree.aggregate_data(), }) } @@ -315,13 +316,13 @@ where } /// Returns the root hash and non-prefixed key of the tree. - pub fn root_hash_key_and_sum(&self) -> CostResult { + pub fn root_hash_key_and_aggregate_data(&self) -> CostResult { self.use_tree(|tree| match tree { - None => Ok((NULL_HASH, None, None)).wrap_with_cost(Default::default()), + None => Ok((NULL_HASH, None, AggregateData::NoAggregateData)).wrap_with_cost(Default::default()), Some(tree) => { - let sum = cost_return_on_error_default!(tree.sum()); + let aggregate_data = cost_return_on_error_default!(tree.aggregate_data()); tree.hash() - .map(|hash| Ok((hash, Some(tree.key().to_vec()), sum))) + .map(|hash| Ok((hash, Some(tree.key().to_vec()), aggregate_data))) } }) } @@ -663,21 +664,21 @@ where skip_sum_checks: bool, grove_version: &GroveVersion, ) { - let (hash, key, sum) = match link { - Link::Reference { hash, key, sum, .. } => { - (hash.to_owned(), key.to_owned(), sum.to_owned()) + let (hash, key, aggregate_data) = match link { + Link::Reference { hash, key, aggregate_data, .. } => { + (hash.to_owned(), key.to_owned(), aggregate_data.to_owned()) } Link::Modified { tree, .. } => ( tree.hash().unwrap(), tree.key().to_vec(), - tree.sum().unwrap(), + tree.aggregate_data().unwrap(), ), Link::Loaded { hash, child_heights: _, - sum, + aggregate_data, tree, - } => (hash.to_owned(), tree.key().to_vec(), sum.to_owned()), + } => (hash.to_owned(), tree.key().to_vec(), aggregate_data.to_owned()), _ => todo!(), }; @@ -711,7 +712,7 @@ where } // Need to skip this when restoring a sum tree - if !skip_sum_checks && node.sum().unwrap() != sum { + if !skip_sum_checks && node.aggregate_data().unwrap() != aggregate_data { bad_link_map.insert(instruction_id.to_vec(), hash); parent_keys.insert(instruction_id.to_vec(), parent_key.to_vec()); return; diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index 1082e80b8..55ee663ec 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -315,11 +315,11 @@ impl<'db, S: StorageContext<'db>> Restorer { .expect("rewrite is only called when traversal_instruction is not empty"); let updated_key = chunk_tree.key(); - let updated_sum = chunk_tree.sum(); + let updated_sum = chunk_tree.aggregate_data(); - if let Some(Link::Reference { key, sum, .. }) = parent.link_mut(*is_left) { + if let Some(Link::Reference { key, aggregate_data, .. }) = parent.link_mut(*is_left) { *key = updated_key.to_vec(); - *sum = updated_sum; + *aggregate_data = updated_sum; } let parent_bytes = parent.encode(); diff --git a/merk/src/proofs/tree.rs b/merk/src/proofs/tree.rs index 16655a6dc..094fe48fa 100644 --- a/merk/src/proofs/tree.rs +++ b/merk/src/proofs/tree.rs @@ -21,6 +21,8 @@ use crate::{ Link, TreeFeatureType::SummedMerkNode, }; +use crate::tree::AggregateData; +use crate::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; #[cfg(any(feature = "full", feature = "verify"))] /// Contains a tree's child node and its hash. The hash can always be assumed to @@ -36,24 +38,20 @@ pub struct Child { impl Child { #[cfg(feature = "full")] pub fn as_link(&self) -> Link { - let (key, sum) = match &self.tree.node { - Node::KV(key, _) | Node::KVValueHash(key, ..) => (key.as_slice(), None), + let (key, aggregate_data) = match &self.tree.node { + Node::KV(key, _) | Node::KVValueHash(key, ..) => (key.as_slice(), AggregateData::NoAggregateData), Node::KVValueHashFeatureType(key, _, _, feature_type) => { - let sum_value = match feature_type { - SummedMerkNode(sum) => Some(*sum), - _ => None, - }; - (key.as_slice(), sum_value) + (key.as_slice(), (*feature_type).into()) } // for the connection between the trunk and leaf chunks, we don't // have the child key so we must first write in an empty one. once // the leaf gets verified, we can write in this key to its parent - _ => (&[] as &[u8], None), + _ => (&[] as &[u8], AggregateData::NoAggregateData), }; Link::Reference { hash: self.hash, - sum, + aggregate_data, child_heights: ( self.tree.child_heights.0 as u8, self.tree.child_heights.1 as u8, @@ -294,12 +292,9 @@ impl Tree { } #[cfg(feature = "full")] - pub(crate) fn sum(&self) -> Option { + pub(crate) fn aggregate_data(&self) -> AggregateData { match self.node { - Node::KVValueHashFeatureType(.., feature_type) => match feature_type { - SummedMerkNode(sum) => Some(sum), - _ => None, - }, + Node::KVValueHashFeatureType(.., feature_type) => feature_type.into(), _ => panic!("Expected node to be type KVValueHashFeatureType"), } } @@ -639,7 +634,7 @@ mod test { left_link, Link::Reference { hash: tree.left.as_ref().map(|node| node.hash).unwrap(), - sum: None, + aggregate_data: None, child_heights: (0, 0), key: vec![1] } @@ -649,7 +644,7 @@ mod test { right_link, Link::Reference { hash: tree.right.as_ref().map(|node| node.hash).unwrap(), - sum: None, + aggregate_data: None, child_heights: (0, 0), key: vec![3] } @@ -688,7 +683,7 @@ mod test { left_link, Link::Reference { hash: tree.left.as_ref().map(|node| node.hash).unwrap(), - sum: Some(3), + aggregate_data: Some(3), child_heights: (0, 0), key: vec![1] } @@ -698,7 +693,7 @@ mod test { right_link, Link::Reference { hash: tree.right.as_ref().map(|node| node.hash).unwrap(), - sum: Some(1), + aggregate_data: Some(1), child_heights: (0, 0), key: vec![3] } diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index cd10937d9..ab8f50718 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -254,7 +254,7 @@ mod tests { [55; 32], Some(Link::Reference { hash: [66; 32], - sum: None, + aggregate_data: None, child_heights: (123, 124), key: vec![2], }), @@ -328,7 +328,7 @@ mod tests { key, child_heights, hash, - sum: _, + aggregate_data: _, }) = tree.link(true) { assert_eq!(*key, [2]); diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index f445dd11b..143207548 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -2,7 +2,7 @@ #[cfg(feature = "full")] use std::io::{Read, Write}; - +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "full")] use ed::{Decode, Encode, Result, Terminated}; #[cfg(feature = "full")] @@ -12,7 +12,8 @@ use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; use super::{hash::CryptoHash, TreeNode}; #[cfg(feature = "full")] use crate::HASH_LENGTH_U32; - +#[cfg(feature = "full")] +use crate::tree::tree_feature_type::AggregateData; // TODO: optimize memory footprint #[cfg(feature = "full")] @@ -30,8 +31,8 @@ pub enum Link { child_heights: (u8, u8), /// Key key: Vec, - /// Sum - sum: Option, + /// Aggregate data like Sum + aggregate_data: AggregateData, }, /// Represents a tree node which has been modified since the `Tree`'s last @@ -57,8 +58,8 @@ pub enum Link { child_heights: (u8, u8), /// Tree tree: TreeNode, - /// Sum - sum: Option, + /// Aggregate data like Sum + aggregate_data: AggregateData, }, /// Represents a tree node which has not been modified, has an up-to-date @@ -70,8 +71,8 @@ pub enum Link { child_heights: (u8, u8), /// Tree tree: TreeNode, - /// Sum - sum: Option, + /// Aggregate data like Sum + aggregate_data: AggregateData, }, } @@ -160,12 +161,12 @@ impl Link { /// of variant `Link::Modified` since we have not yet recomputed the tree's /// hash. #[inline] - pub const fn sum(&self) -> Option { + pub const fn aggregateData(&self) -> AggregateData { match self { Link::Modified { .. } => panic!("Cannot get hash from modified link"), - Link::Reference { sum, .. } => *sum, - Link::Uncommitted { sum, .. } => *sum, - Link::Loaded { sum, .. } => *sum, + Link::Reference { aggregate_data, .. } => *aggregate_data, + Link::Uncommitted { aggregate_data, .. } => *aggregate_data, + Link::Loaded { aggregate_data, .. } => *aggregate_data, } } @@ -213,12 +214,12 @@ impl Link { Link::Uncommitted { .. } => panic!("Cannot prune Uncommitted tree"), Link::Loaded { hash, - sum, + aggregate_data, child_heights, tree, } => Self::Reference { hash, - sum, + aggregate_data, child_heights, key: tree.take_key(), }, @@ -269,9 +270,9 @@ impl Link { debug_assert!(self.key().len() < 256, "Key length must be less than 256"); Ok(match self { - Link::Reference { key, sum, .. } => match sum { - None => key.len() + 36, // 1 + HASH_LENGTH + 2 + 1, - Some(_sum_value) => { + Link::Reference { key, aggregate_data, .. } => match aggregate_data { + AggregateData::NoAggregateData => key.len() + 36, // 1 + HASH_LENGTH + 2 + 1, + AggregateData::Count(_) | AggregateData::Sum(_) => { // 1 for key len // key_len for keys // 32 for hash @@ -282,14 +283,27 @@ impl Link { // sum_len for sum vale key.len() + 44 // 1 + 32 + 2 + 1 + 8 } + AggregateData::BigSum(_) => { + // 1 for key len + // key_len for keys + // 32 for hash + // 2 for child heights + // 1 to represent presence of sum value + // if above is 1, then + // 1 for sum len + // sum_len for sum vale + key.len() + 52 // 1 + 32 + 2 + 1 + 16 + } }, Link::Modified { .. } => panic!("No encoding for Link::Modified"), - Link::Uncommitted { tree, sum, .. } | Link::Loaded { tree, sum, .. } => match sum { - None => tree.key().len() + 36, // 1 + 32 + 2 + 1, - Some(sum_value) => { - let _encoded_sum_value = sum_value.encode_var_vec(); + Link::Uncommitted { tree, aggregate_data, .. } | Link::Loaded { tree, aggregate_data, .. } => match aggregate_data { + AggregateData::NoAggregateData => tree.key().len() + 36, // 1 + 32 + 2 + 1, + AggregateData::Count(_) | AggregateData::Sum(_) => { tree.key().len() + 44 // 1 + 32 + 2 + 1 + 8 } + AggregateData::BigSum(_) => { + tree.key().len() + 52 // 1 + 32 + 2 + 1 + 16 + } }, }) } @@ -299,25 +313,25 @@ impl Link { impl Encode for Link { #[inline] fn encode_into(&self, out: &mut W) -> Result<()> { - let (hash, sum, key, (left_height, right_height)) = match self { + let (hash, aggregate_data, key, (left_height, right_height)) = match self { Link::Reference { hash, - sum, + aggregate_data, key, child_heights, - } => (hash, sum, key.as_slice(), child_heights), + } => (hash, aggregate_data, key.as_slice(), child_heights), Link::Loaded { hash, - sum, + aggregate_data, tree, child_heights, - } => (hash, sum, tree.key(), child_heights), + } => (hash, aggregate_data, tree.key(), child_heights), Link::Uncommitted { hash, - sum, + aggregate_data, tree, child_heights, - } => (hash, sum, tree.key(), child_heights), + } => (hash, aggregate_data, tree.key(), child_heights), Link::Modified { .. } => panic!("No encoding for Link::Modified"), }; @@ -331,13 +345,21 @@ impl Encode for Link { out.write_all(&[*left_height, *right_height])?; - match sum { - None => { + match aggregate_data { + AggregateData::NoAggregateData => { out.write_all(&[0])?; } - Some(sum_value) => { + AggregateData::Sum(sum_value) => { out.write_all(&[1])?; - out.write_varint(sum_value.to_owned())?; + out.write_varint(*sum_value)?; + } + AggregateData::BigSum(big_sum_value) => { + out.write_all(&[2])?; + out.write_i128::(*big_sum_value)?; + } + AggregateData::Count(count_value) => { + out.write_all(&[2])?; + out.write_varint(*count_value)?; } } @@ -349,9 +371,9 @@ impl Encode for Link { debug_assert!(self.key().len() < 256, "Key length must be less than 256"); Ok(match self { - Link::Reference { key, sum, .. } => match sum { - None => key.len() + 36, // 1 + 32 + 2 + 1 - Some(sum_value) => { + Link::Reference { key, aggregate_data, .. } => match aggregate_data { + AggregateData::NoAggregateData => key.len() + 36, // 1 + 32 + 2 + 1 + AggregateData::Sum(sum_value) => { let encoded_sum_value = sum_value.encode_var_vec(); // 1 for key len // key_len for keys @@ -363,14 +385,44 @@ impl Encode for Link { // sum_len for sum vale key.len() + encoded_sum_value.len() + 36 // 1 + 32 + 2 + 1 } + AggregateData::BigSum(_) => { + // 1 for key len + // key_len for keys + // 32 for hash + // 2 for child heights + // 1 to represent presence of sum value + // if above is 1, then + // 1 for sum len + // sum_len for sum vale + key.len() + 52 // 1 + 32 + 2 + 1 + 16 + } + AggregateData::Count(count) => { + let encoded_sum_value = count.encode_var_vec(); + // 1 for key len + // key_len for keys + // 32 for hash + // 2 for child heights + // 1 to represent presence of sum value + // if above is 1, then + // 1 for sum len + // sum_len for sum vale + key.len() + encoded_sum_value.len() + 36 // 1 + 32 + 2 + 1 + } }, Link::Modified { .. } => panic!("No encoding for Link::Modified"), - Link::Uncommitted { tree, sum, .. } | Link::Loaded { tree, sum, .. } => match sum { - None => tree.key().len() + 36, // 1 + 32 + 2 + 1 - Some(sum_value) => { + Link::Uncommitted { tree, aggregate_data, .. } | Link::Loaded { tree, aggregate_data, .. } => match aggregate_data { + AggregateData::NoAggregateData => tree.key().len() + 36, // 1 + 32 + 2 + 1 + AggregateData::Sum(sum_value) => { let encoded_sum_value = sum_value.encode_var_vec(); tree.key().len() + encoded_sum_value.len() + 36 // 1 + 32 + 2 + 1 } + AggregateData::BigSum(_) => { + tree.key().len() + 52 // 1 + 32 + 2 + 1 + 16 + } + AggregateData::Count(count_value) => { + let encoded_count_value = count_value.encode_var_vec(); + tree.key().len() + encoded_count_value.len() + 36 // 1 + 32 + 2 + 1 + } }, }) } @@ -383,7 +435,7 @@ impl Link { Self::Reference { key: Vec::with_capacity(64), hash: Default::default(), - sum: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (0, 0), } } @@ -407,7 +459,7 @@ impl Decode for Link { } if let Link::Reference { - ref mut sum, + ref mut aggregate_data, ref mut key, ref mut hash, ref mut child_heights, @@ -423,12 +475,20 @@ impl Decode for Link { child_heights.0 = read_u8(&mut input)?; child_heights.1 = read_u8(&mut input)?; - let has_sum = read_u8(&mut input)?; - *sum = match has_sum { - 0 => None, + let aggregate_data_byte = read_u8(&mut input)?; + *aggregate_data = match aggregate_data_byte { + 0 => AggregateData::NoAggregateData, 1 => { let encoded_sum: i64 = input.read_varint()?; - Some(encoded_sum) + AggregateData::Sum(encoded_sum) + } + 2 => { + let encoded_big_sum: i128 = input.read_i128::()?; + AggregateData::BigSum(encoded_big_sum) + } + 3 => { + let encoded_count: u64 = input.read_varint()?; + AggregateData::Count(encoded_count) } _ => return Err(ed::Error::UnexpectedByte(55)), }; @@ -487,7 +547,7 @@ mod test { #[test] fn types() { let hash = NULL_HASH; - let sum = None; + let aggregate_data = AggregateData::NoAggregateData; let child_heights = (0, 0); let pending_writes = 1; let key = vec![0]; @@ -495,7 +555,7 @@ mod test { let reference = Link::Reference { hash, - sum, + aggregate_data, child_heights, key, }; @@ -506,13 +566,13 @@ mod test { }; let uncommitted = Link::Uncommitted { hash, - sum, + aggregate_data, child_heights, tree: tree(), }; let loaded = Link::Loaded { hash, - sum, + aggregate_data, child_heights, tree: tree(), }; @@ -578,7 +638,7 @@ mod test { fn uncommitted_into_reference() { Link::Uncommitted { hash: [1; 32], - sum: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (1, 1), tree: TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(), } @@ -589,7 +649,7 @@ mod test { fn encode_link() { let link = Link::Reference { key: vec![1, 2, 3], - sum: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (123, 124), hash: [55; 32], }; @@ -610,7 +670,53 @@ mod test { fn encode_link_with_sum() { let link = Link::Reference { key: vec![1, 2, 3], - sum: Some(50), + aggregate_data: AggregateData::Sum(50), + child_heights: (123, 124), + hash: [55; 32], + }; + assert_eq!(link.encoding_length().unwrap(), 40); + + let mut bytes = vec![]; + link.encode_into(&mut bytes).unwrap(); + + assert_eq!(link.encoding_length().unwrap(), bytes.len()); + assert_eq!( + bytes, + vec![ + 3, 1, 2, 3, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 1, 100, + ] + ); + } + + #[test] + fn encode_link_with_count() { + let link = Link::Reference { + key: vec![1, 2, 3], + aggregate_data: AggregateData::Count(50), + child_heights: (123, 124), + hash: [55; 32], + }; + assert_eq!(link.encoding_length().unwrap(), 40); + + let mut bytes = vec![]; + link.encode_into(&mut bytes).unwrap(); + + assert_eq!(link.encoding_length().unwrap(), bytes.len()); + assert_eq!( + bytes, + vec![ + 3, 1, 2, 3, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 1, 100, + ] + ); + } + + #[test] + fn encode_link_with_big_sum() { + let link = Link::Reference { + key: vec![1, 2, 3], + aggregate_data: AggregateData::BigSum(50), child_heights: (123, 124), hash: [55; 32], }; @@ -634,7 +740,7 @@ mod test { fn encode_link_long_key() { let link = Link::Reference { key: vec![123; 300], - sum: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (123, 124), hash: [55; 32], }; @@ -649,6 +755,6 @@ mod test { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 0, ]; let link = Link::decode(bytes.as_slice()).expect("expected to decode a link"); - assert_eq!(link.sum(), None); + assert_eq!(link.aggregateData(), AggregateData::NoAggregateData); } } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 91eebf52a..084f3059f 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -61,6 +61,8 @@ pub use link::Link; pub use ops::{AuxMerkBatch, BatchEntry, MerkBatch, Op, PanicSource}; #[cfg(any(feature = "full", feature = "verify"))] pub use tree_feature_type::TreeFeatureType; +#[cfg(any(feature = "full", feature = "verify"))] +pub use tree_feature_type::AggregateData; #[cfg(feature = "full")] pub use walk::{Fetch, RefWalker, Walker}; @@ -447,9 +449,12 @@ impl TreeNode { ( // 36 = 32 Hash + 1 key length + 2 child heights + 1 feature type link.key().len() as u32 + 36, - link.sum() - .map(|s| s.encode_var_vec().len() as u32) - .unwrap_or_default(), + match link.aggregateData() { + AggregateData::NoAggregateData => 0, + AggregateData::Sum(s) => s.encode_var_vec().len() as u32, + AggregateData::BigSum(s) => 16 as u32, + AggregateData::Count(c) => c.encode_var_vec().len() as u32, + } ) }) } @@ -490,9 +495,56 @@ impl TreeNode { /// Returns the sum of the root node's child on the given side, if any. If /// there is no child, returns 0. #[inline] - pub fn child_sum(&self, left: bool) -> i64 { + pub fn child_aggregate_data_as_i64(&self, left: bool) -> Result { + match self.link(left) { + Some(link) => match link.aggregateData() { + AggregateData::NoAggregateData => Ok(0), + AggregateData::Sum(s) => Ok(s), + AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree("for aggregate data as i64".to_string())), + AggregateData::Count(c) => { + if c > i64::MAX as u64 { + Err(Overflow("count overflow when below sum tree")) + } else { + Ok(c as i64) + } + } + } + _ => Ok(0), + } + } + + /// Returns the sum of the root node's child on the given side, if any. If + /// there is no child, returns 0. + #[inline] + pub fn child_aggregate_data_as_u64(&self, left: bool) -> Result { + match self.link(left) { + Some(link) => match link.aggregateData() { + AggregateData::NoAggregateData => Ok(0), + AggregateData::Sum(s) => { + if s < 0 { + Err(Error::Overflow("negative sum tree under count tree")) + } else { + Ok(s as u64) + } + }, + AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree("for aggregate data as u64".to_string())), + AggregateData::Count(c) => Ok(c) + } + _ => Ok(0), + } + } + + /// Returns the sum of the root node's child on the given side, if any. If + /// there is no child, returns 0. + #[inline] + pub fn child_aggregate_data_as_i128(&self, left: bool) -> i128 { match self.link(left) { - Some(link) => link.sum().unwrap_or_default(), + Some(link) => match link.aggregateData() { + AggregateData::NoAggregateData => 0, + AggregateData::Sum(s) => s as i128, + AggregateData::BigSum(s) => s, + AggregateData::Count(c) => c as i128, + } _ => 0, } } @@ -510,14 +562,32 @@ impl TreeNode { /// Computes and returns the hash of the root node. #[inline] - pub fn sum(&self) -> Result, Error> { + pub fn aggregate_data(&self) -> Result { match self.inner.kv.feature_type { - TreeFeatureType::BasicMerkNode => Ok(None), - TreeFeatureType::SummedMerkNode(value) => value - .checked_add(self.child_sum(true)) - .and_then(|a| a.checked_add(self.child_sum(false))) - .ok_or(Overflow("sum is overflowing")) - .map(Some), + TreeFeatureType::BasicMerkNode => Ok(AggregateData::NoAggregateData), + TreeFeatureType::SummedMerkNode(value) => { + let left = self.child_aggregate_data_as_i64(true)?; + let right = self.child_aggregate_data_as_i64(false)?; + value + .checked_add(left) + .and_then(|a| a.checked_add(right)) + .ok_or(Overflow("sum is overflowing")) + .map(AggregateData::Sum) + }, + TreeFeatureType::BigSummedMerkNode(value) => value + .checked_add(self.child_aggregate_data_as_i128(true)) + .and_then(|a| a.checked_add(self.child_aggregate_data_as_i128(false))) + .ok_or(Overflow("big sum is overflowing")) + .map(AggregateData::BigSum), + TreeFeatureType::CountedMerkNode(value) => { + let left = self.child_aggregate_data_as_u64(true)?; + let right = self.child_aggregate_data_as_u64(false)?; + value + .checked_add(left) + .and_then(|a| a.checked_add(right)) + .ok_or(Overflow("count is overflowing")) + .map(AggregateData::Count) + }, } } @@ -936,13 +1006,13 @@ impl TreeNode { { // println!("key is {}", std::str::from_utf8(tree.key()).unwrap()); cost_return_on_error!(&mut cost, tree.commit(c, old_specialized_cost,)); - let sum = cost_return_on_error_default!(tree.sum()); + let aggregate_data = cost_return_on_error_default!(tree.aggregate_data()); self.inner.left = Some(Link::Loaded { hash: tree.hash().unwrap_add_cost(&mut cost), tree, child_heights, - sum, + aggregate_data, }); } else { unreachable!() @@ -959,12 +1029,12 @@ impl TreeNode { { // println!("key is {}", std::str::from_utf8(tree.key()).unwrap()); cost_return_on_error!(&mut cost, tree.commit(c, old_specialized_cost,)); - let sum = cost_return_on_error_default!(tree.sum()); + let aggregate_data = cost_return_on_error_default!(tree.aggregate_data()); self.inner.right = Some(Link::Loaded { hash: tree.hash().unwrap_add_cost(&mut cost), tree, child_heights, - sum, + aggregate_data, }); } else { unreachable!() @@ -1001,13 +1071,13 @@ impl TreeNode { { // TODO: return Err instead of panic? let link = self.link(left).expect("Expected link"); - let (child_heights, hash, sum) = match link { + let (child_heights, hash, aggregate_data) = match link { Link::Reference { child_heights, hash, - sum, + aggregate_data, .. - } => (child_heights, hash, sum), + } => (child_heights, hash, aggregate_data), _ => panic!("Expected Some(Link::Reference)"), }; @@ -1021,7 +1091,7 @@ impl TreeNode { tree, hash: *hash, child_heights: *child_heights, - sum: *sum, + aggregate_data: *aggregate_data, }); Ok(()).wrap_with_cost(cost) } diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index c47fb0d60..e5f0e2763 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -2,7 +2,7 @@ #[cfg(any(feature = "full", feature = "verify"))] use std::io::{Read, Write}; - +use byteorder::{BigEndian, ByteOrder, LittleEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "full")] use ed::Terminated; #[cfg(any(feature = "full", feature = "verify"))] @@ -12,6 +12,7 @@ use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(any(feature = "full", feature = "verify"))] use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; +use crate::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -21,6 +22,30 @@ pub enum TreeFeatureType { BasicMerkNode, /// Summed Merk Tree Node SummedMerkNode(i64), + /// Big Summed Merk Tree Node + BigSummedMerkNode(i128), + /// Counted Merk Tree None + CountedMerkNode(u64), +} + +#[cfg(any(feature = "full", feature = "verify"))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum AggregateData { + NoAggregateData, + Sum(i64), + BigSum(i128), + Count(u64), +} + +impl From for AggregateData { + fn from(value: TreeFeatureType) -> Self { + match value { + BasicMerkNode => AggregateData::NoAggregateData, + SummedMerkNode(val) => AggregateData::Sum(val), + BigSummedMerkNode(val) => AggregateData::BigSum(val), + CountedMerkNode(val) => AggregateData::Count(val), + } + } } #[cfg(feature = "full")] @@ -31,6 +56,8 @@ impl TreeFeatureType { match self { BasicMerkNode => None, SummedMerkNode(m) => Some(m.encode_var_vec().len() as u32), + BigSummedMerkNode(_) => Some(16), + CountedMerkNode(m) => Some(m.encode_var_vec().len() as u32), } } @@ -46,6 +73,8 @@ impl TreeFeatureType { match self { BasicMerkNode => 1, SummedMerkNode(_sum) => 9, + BigSummedMerkNode(_) => 17, + CountedMerkNode(_) => 9, } } } @@ -63,7 +92,17 @@ impl Encode for TreeFeatureType { } SummedMerkNode(sum) => { dest.write_all(&[1])?; - dest.write_varint(sum.to_owned())?; + dest.write_varint(*sum)?; + Ok(()) + } + BigSummedMerkNode(sum) => { + dest.write_all(&[2])?; + dest.write_i128::(*sum)?; + Ok(()) + } + CountedMerkNode(count) => { + dest.write_all(&[3])?; + dest.write_varint(*count)?; Ok(()) } } @@ -79,6 +118,15 @@ impl Encode for TreeFeatureType { // encoded_sum.len() for the length of the encoded vector Ok(1 + encoded_sum.len()) } + BigSummedMerkNode(_) => { + Ok(17) + } + CountedMerkNode(count) => { + let encoded_sum = count.encode_var_vec(); + // 1 for the enum type + // encoded_sum.len() for the length of the encoded vector + Ok(1 + encoded_sum.len()) + } } } } @@ -95,6 +143,14 @@ impl Decode for TreeFeatureType { let encoded_sum: i64 = input.read_varint()?; Ok(SummedMerkNode(encoded_sum)) } + [2] => { + let encoded_sum: i128 = input.read_i128::()?; + Ok(BigSummedMerkNode(encoded_sum)) + } + [3] => { + let encoded_count: u64 = input.read_varint()?; + Ok(CountedMerkNode(encoded_count)) + } _ => Err(ed::Error::UnexpectedByte(55)), } } diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 4b67bb609..2274f804f 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -491,7 +491,7 @@ mod test { hash: Default::default(), key: b"foo".to_vec(), child_heights: (0, 0), - sum: None, + aggregate_data: None, }), None, BasicMerkNode, From b3e287b221088c4855434e14a8d15615739743a7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 7 Jan 2025 14:23:12 +0700 Subject: [PATCH 02/30] more work --- .../batch/just_in_time_reference_update.rs | 11 +-- grovedb/src/batch/mod.rs | 51 +++++------ grovedb/src/element/constructor.rs | 26 ++++++ grovedb/src/element/delete.rs | 4 +- grovedb/src/element/helpers.rs | 59 ++++++++++-- grovedb/src/element/insert.rs | 14 +-- grovedb/src/element/mod.rs | 32 +++++++ .../src/estimated_costs/average_case_costs.rs | 8 +- grovedb/src/lib.rs | 85 ++++++++++++------ grovedb/src/operations/delete/mod.rs | 4 +- grovedb/src/tests/sum_tree_tests.rs | 21 ++--- .../src/estimated_costs/average_case_costs.rs | 42 ++++----- merk/src/estimated_costs/mod.rs | 42 +++++---- merk/src/estimated_costs/worst_case_costs.rs | 19 ++-- merk/src/merk/apply.rs | 5 +- merk/src/merk/mod.rs | 51 ++++++++++- merk/src/merk/open.rs | 26 +++--- merk/src/merk/restore.rs | 22 ++--- merk/src/merk/source.rs | 7 +- merk/src/test_utils/mod.rs | 21 ++--- merk/src/test_utils/temp_merk.rs | 5 +- merk/src/tree/encoding.rs | 7 +- merk/src/tree/kv.rs | 47 +++++----- merk/src/tree/link.rs | 5 +- merk/src/tree/mod.rs | 9 +- merk/src/tree/tree_feature_type.rs | 89 +++++++++++++++++-- 26 files changed, 498 insertions(+), 214 deletions(-) diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index f4385b899..9fb1ec158 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -12,6 +12,7 @@ use grovedb_merk::{ tree::{kv::KV, value_hash, TreeNode}, CryptoHash, Merk, }; +use grovedb_merk::merk::TreeType; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; @@ -31,7 +32,7 @@ where new_element: &mut Element, old_element: Element, old_serialized_element: &[u8], - is_in_sum_tree: bool, + in_tree_type: TreeType, flags_update: &mut G, split_removal_bytes: &mut SR, grove_version: &GroveVersion, @@ -79,7 +80,7 @@ where let old_storage_cost = KV::node_value_byte_cost_size( key.len() as u32, old_serialized_element.len() as u32, - is_in_sum_tree, + in_tree_type.inner_node_type(), ); let original_new_element = new_element.clone(); @@ -99,10 +100,10 @@ where KV::node_value_byte_cost_size( key.len() as u32, serialized_with_old_flags.len() as u32, - is_in_sum_tree, + in_tree_type.inner_node_type(), ) } else { - KV::node_value_byte_cost_size(key.len() as u32, serialized.len() as u32, is_in_sum_tree) + KV::node_value_byte_cost_size(key.len() as u32, serialized.len() as u32, in_tree_type.inner_node_type()) }; let mut i = 0; @@ -153,7 +154,7 @@ where new_storage_cost = KV::node_value_byte_cost_size( key.len() as u32, new_serialized_bytes.len() as u32, - is_in_sum_tree, + in_tree_type.inner_node_type(), ); if serialization_to_use == new_serialized_bytes { diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 13525edaf..106ab910b 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -66,6 +66,7 @@ use grovedb_version::{ use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; +use grovedb_merk::tree::AggregateData; use key_info::{KeyInfo, KeyInfo::KnownKey}; pub use options::BatchApplyOptions; @@ -91,8 +92,8 @@ pub enum GroveOp { hash: [u8; 32], /// Root key root_key: Option>, - /// Sum - sum: Option, + /// Aggregate data + aggregate_data: AggregateData, }, /// Inserts an element that is known to not yet exist InsertOnly { @@ -124,8 +125,8 @@ pub enum GroveOp { root_key: Option>, /// Flags flags: Option, - /// Sum - sum: Option, + /// Aggregate Data such as sum + aggregate_data: AggregateData, }, /// Refresh the reference with information provided /// Providing this information is necessary to be able to calculate @@ -905,7 +906,7 @@ where .map_err(|e| Error::CorruptedData(e.to_string())) ); - let is_sum_tree = merk.is_sum_tree; + let is_sum_tree = merk.tree_type; if let Some(referenced_element) = referenced_element { let element = cost_return_on_error_no_add!( @@ -1240,7 +1241,7 @@ where if let HashMapEntry::Vacant(e) = self.merks.entry(inserted_path.clone()) { let mut merk = cost_return_on_error!(&mut cost, (self.get_merk_fn)(&inserted_path, true)); - merk.is_sum_tree = is_sum_tree; + merk.tree_type = is_sum_tree; e.insert(merk); } @@ -1284,7 +1285,7 @@ where let path = &p; // This also populates Merk trees cache - let is_sum_tree = { + let tree_type = { let merk = match self.merks.entry(path.to_vec()) { HashMapEntry::Occupied(o) => o.into_mut(), HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( @@ -1292,7 +1293,7 @@ where (self.get_merk_fn)(path, false) )), }; - merk.is_sum_tree + merk.tree_type }; let mut batch_operations: Vec<(Vec, Op)> = vec![]; @@ -1306,7 +1307,7 @@ where let merk_feature_type = cost_return_on_error!( &mut cost, element - .get_feature_type(is_sum_tree) + .get_feature_type(tree_type) .wrap_with_cost(OperationCost::default()) ); let path_reference = cost_return_on_error!( @@ -1352,7 +1353,7 @@ where let merk_feature_type = cost_return_on_error!( &mut cost, element - .get_feature_type(is_sum_tree) + .get_feature_type(tree_type) .wrap_with_cost(OperationCost::default()) ); cost_return_on_error!( @@ -1371,7 +1372,7 @@ where let merk_feature_type = cost_return_on_error!( &mut cost, element - .get_feature_type(is_sum_tree) + .get_feature_type(tree_type) .wrap_with_cost(OperationCost::default()) ); if batch_apply_options.validate_insertion_does_not_override { @@ -1450,7 +1451,7 @@ where .wrap_with_cost(cost); }; - let merk_feature_type = if is_sum_tree { + let merk_feature_type = if tree_type { SummedMerkNode(0) } else { BasicMerkNode @@ -1501,7 +1502,7 @@ where Element::delete_into_batch_operations( key_info.get_key(), false, - is_sum_tree, /* we are in a sum tree, this might or might not be a + tree_type, /* we are in a sum tree, this might or might not be a * sum item */ &mut batch_operations, grove_version @@ -1535,7 +1536,7 @@ where GroveOp::ReplaceTreeRootKey { hash, root_key, - sum, + aggregate_data, } => { let merk = self.merks.get(path).expect("the Merk is cached"); cost_return_on_error!( @@ -1545,7 +1546,7 @@ where key_info.get_key(), root_key, hash, - sum, + aggregate_data, &mut batch_operations, grove_version ) @@ -1564,7 +1565,7 @@ where ), }; let merk_feature_type = - cost_return_on_error_no_add!(&cost, element.get_feature_type(is_sum_tree)); + cost_return_on_error_no_add!(&cost, element.get_feature_type(tree_type)); cost_return_on_error!( &mut cost, @@ -1590,7 +1591,7 @@ where &[], Some(batch_apply_options.as_merk_options()), &|key, value| { - Element::specialized_costs_for_key_value(key, value, is_sum_tree, grove_version) + Element::specialized_costs_for_key_value(key, value, tree_type, grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), @@ -1776,7 +1777,7 @@ impl GroveDb { ); } } else { - let (root_hash, calculated_root_key, sum_value) = cost_return_on_error!( + let (root_hash, calculated_root_key, aggregate_data) = cost_return_on_error!( &mut cost, merk_tree_cache.execute_ops_on_path( &path, @@ -1806,7 +1807,7 @@ impl GroveDb { GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, - sum: sum_value, + aggregate_data, } .into(), ); @@ -1817,11 +1818,11 @@ impl GroveDb { GroveOp::ReplaceTreeRootKey { hash, root_key, - sum, + aggregate_data, } => { *hash = root_hash; *root_key = calculated_root_key; - *sum = sum_value; + *aggregate_data = aggregate_data; } GroveOp::InsertTreeWithRootHash { .. } => { return Err(Error::CorruptedCodeExecution( @@ -1839,7 +1840,7 @@ impl GroveDb { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), - sum: None, + aggregate_data: AggregateData::NoAggregateData, } .into(); } else if let Element::SumTree(.., flags) = @@ -1850,7 +1851,7 @@ impl GroveDb { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), - sum: sum_value, + aggregate_data, } .into(); } else { @@ -1889,7 +1890,7 @@ impl GroveDb { GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, - sum: sum_value, + aggregate_data, }, ); ops_at_level_above.insert(parent_path, ops_on_path); @@ -1901,7 +1902,7 @@ impl GroveDb { GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, - sum: sum_value, + aggregate_data, } .into(), ); diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 91143ec87..75e051f54 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -7,6 +7,7 @@ use crate::{ reference_path::ReferencePathType, Element, ElementFlags, }; +use crate::element::BigSumValue; impl Element { #[cfg(feature = "full")] @@ -131,4 +132,29 @@ impl Element { ) -> Self { Element::SumTree(maybe_root_key, sum_value, flags) } + + #[cfg(feature = "full")] + /// Set element to a big sum tree without flags + pub fn new_big_sum_tree(maybe_root_key: Option>) -> Self { + Element::BigSumTree(maybe_root_key, 0, None) + } + + #[cfg(feature = "full")] + /// Set element to a sum tree with flags + pub fn new_big_sum_tree_with_flags( + maybe_root_key: Option>, + flags: Option, + ) -> Self { + Element::BigSumTree(maybe_root_key, 0, flags) + } + + #[cfg(feature = "full")] + /// Set element to a sum tree with flags and sum value + pub fn new_big_sum_tree_with_flags_and_sum_value( + maybe_root_key: Option>, + big_sum_value: BigSumValue, + flags: Option, + ) -> Self { + Element::BigSumTree(maybe_root_key, big_sum_value, flags) + } } diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index ced24e273..9382c435a 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -38,7 +38,7 @@ impl Element { (false, false) => Op::Delete, }; let batch = [(key, op)]; - let uses_sum_nodes = merk.is_sum_tree; + let uses_sum_nodes = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch, &[], @@ -85,7 +85,7 @@ impl Element { (false, false) => Op::Delete, }; let batch = [(key, op)]; - let uses_sum_nodes = merk.is_sum_tree; + let uses_sum_nodes = merk.tree_type; merk.apply_with_costs_just_in_time_value_update::<_, Vec>( &batch, &[], diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 5b3662dff..20022d6fd 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -16,7 +16,8 @@ use grovedb_merk::{ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; - +use grovedb_merk::merk::TreeType; +use grovedb_merk::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; #[cfg(any(feature = "full", feature = "verify"))] @@ -28,6 +29,7 @@ use crate::{ }; #[cfg(any(feature = "full", feature = "verify"))] use crate::{Element, Error}; +use crate::element::BIG_SUM_TREE_COST_SIZE; impl Element { #[cfg(any(feature = "full", feature = "verify"))] @@ -40,6 +42,17 @@ impl Element { } } + #[cfg(any(feature = "full", feature = "verify"))] + /// Decoded the integer value in the SumItem element type, returns 0 for + /// everything else + pub fn big_sum_value_or_default(&self) -> i128 { + match self { + Element::SumItem(sum_value, _) | Element::SumTree(_, sum_value, _) => *sum_value as i128, + Element::BigSumTree(_, sum_value, _) => *sum_value, + _ => 0, + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Decoded the integer value in the SumItem element type pub fn as_sum_item_value(&self) -> Result { @@ -109,6 +122,23 @@ impl Element { matches!(self, Element::SumTree(..)) } + #[cfg(any(feature = "full", feature = "verify"))] + /// Check if the element is a tree and return the tree type + pub fn tree_type(&self) -> Option { + match self { + Element::Tree(_, _) => Some(TreeType::NormalTree), + Element::SumTree(_, _, _) => Some(TreeType::SumTree), + Element::BigSumTree(_, _, _) => Some(TreeType::BigSumTree), + _ => None, + } + } + + #[cfg(any(feature = "full", feature = "verify"))] + /// Check if the element is a big sum tree + pub fn is_big_sum_tree(&self) -> bool { + matches!(self, Element::BigSumTree(..)) + } + #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a tree but not a sum tree pub fn is_basic_tree(&self) -> bool { @@ -118,7 +148,7 @@ impl Element { #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a tree pub fn is_any_tree(&self) -> bool { - matches!(self, Element::SumTree(..) | Element::Tree(..)) + matches!(self, Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..)) } #[cfg(any(feature = "full", feature = "verify"))] @@ -147,10 +177,12 @@ impl Element { #[cfg(feature = "full")] /// Get the tree feature type - pub fn get_feature_type(&self, parent_is_sum_tree: bool) -> Result { - match parent_is_sum_tree { - true => Ok(SummedMerkNode(self.sum_value_or_default())), - false => Ok(BasicMerkNode), + pub fn get_feature_type(&self, parent_tree_type: TreeType) -> Result { + match parent_tree_type { + TreeType::NormalTree => Ok(BasicMerkNode), + TreeType::SumTree => Ok(SummedMerkNode(self.sum_value_or_default())), + TreeType::BigSumTree => Ok(BigSummedMerkNode(self.big_sum_value_or_default())), + TreeType::CountTree => Ok(CountedMerkNode(1)), } } @@ -291,6 +323,19 @@ impl Element { is_sum_node, ) } + Element::BigSumTree(_, _sum_value, flags) => { + let flags_len = flags.map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let value_len = BIG_SUM_TREE_COST_SIZE + flags_len; + let key_len = key.len() as u32; + KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key_len, + value_len, + is_sum_node, + ) + } Element::SumItem(.., flags) => { let flags_len = flags.map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -315,6 +360,7 @@ impl Element { match self { Element::Tree(..) => Ok(TREE_COST_SIZE), Element::SumTree(..) => Ok(SUM_TREE_COST_SIZE), + Element::BigSumTree(..) => Ok(BIG_SUM_TREE_COST_SIZE), Element::SumItem(..) => Ok(SUM_ITEM_COST_SIZE), _ => Err(Error::CorruptedCodeExecution( "trying to get tree cost from non tree element", @@ -337,6 +383,7 @@ impl Element { match self { Element::Tree(..) => Some(LayeredValueDefinedCost(cost)), Element::SumTree(..) => Some(LayeredValueDefinedCost(cost)), + Element::BigSumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::SumItem(..) => Some(SpecializedValueDefinedCost(cost)), _ => None, } diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index ce0144a2b..97a218a89 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -32,13 +32,13 @@ impl Element { let serialized = cost_return_on_error_default!(self.serialize(grove_version)); - if !merk.is_sum_tree && self.is_sum_item() { + if !merk.tree_type && self.is_sum_item() { return Err(Error::InvalidInput("cannot add sum item to non sum tree")) .wrap_with_cost(Default::default()); } let merk_feature_type = - cost_return_on_error_default!(self.get_feature_type(merk.is_sum_tree)); + cost_return_on_error_default!(self.get_feature_type(merk.tree_type)); let batch_operations = if matches!(self, SumItem(..)) { let value_cost = cost_return_on_error_default!(self.get_specialized_cost(grove_version)); @@ -55,7 +55,7 @@ impl Element { } else { [(key, Op::Put(serialized, merk_feature_type))] }; - let uses_sum_nodes = merk.is_sum_tree; + let uses_sum_nodes = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch_operations, &[], @@ -306,7 +306,7 @@ impl Element { let mut cost = OperationCost::default(); let merk_feature_type = cost_return_on_error!( &mut cost, - self.get_feature_type(merk.is_sum_tree) + self.get_feature_type(merk.tree_type) .wrap_with_cost(OperationCost::default()) ); @@ -314,7 +314,7 @@ impl Element { key, Op::PutCombinedReference(serialized, referenced_value, merk_feature_type), )]; - let uses_sum_nodes = merk.is_sum_tree; + let uses_sum_nodes = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch_operations, &[], @@ -387,7 +387,7 @@ impl Element { let cost = OperationCost::default(); let merk_feature_type = - cost_return_on_error_no_add!(&cost, self.get_feature_type(merk.is_sum_tree)); + cost_return_on_error_no_add!(&cost, self.get_feature_type(merk.tree_type)); let tree_cost = cost_return_on_error_no_add!(&cost, self.get_specialized_cost(grove_version)); @@ -401,7 +401,7 @@ impl Element { key, Op::PutLayeredReference(serialized, cost, subtree_root_hash, merk_feature_type), )]; - let uses_sum_nodes = merk.is_sum_tree; + let uses_sum_nodes = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch_operations, &[], diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 9986c6244..6abdaa19c 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -28,6 +28,7 @@ use bincode::{Decode, Encode}; use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; #[cfg(feature = "full")] use grovedb_merk::estimated_costs::{LAYER_COST_SIZE, SUM_LAYER_COST_SIZE}; +use grovedb_merk::estimated_costs::{BIG_SUM_LAYER_COST_SIZE, BIG_SUM_VALUE_EXTRA_COST}; #[cfg(feature = "full")] use grovedb_visualize::visualize_to_vec; @@ -60,10 +61,22 @@ pub const SUM_ITEM_COST_SIZE: u32 = SUM_VALUE_EXTRA_COST + 2; // 11 /// The cost of a sum tree pub const SUM_TREE_COST_SIZE: u32 = SUM_LAYER_COST_SIZE; // 12 +#[cfg(feature = "full")] +/// The cost of a big sum tree +pub const BIG_SUM_TREE_COST_SIZE: u32 = BIG_SUM_LAYER_COST_SIZE; // 19 + #[cfg(any(feature = "full", feature = "verify"))] /// int 64 sum value pub type SumValue = i64; +#[cfg(any(feature = "full", feature = "verify"))] +/// int 128 sum value +pub type BigSumValue = i128; + +#[cfg(any(feature = "full", feature = "verify"))] +/// int 64 count value +pub type CountValue = u64; + #[cfg(any(feature = "full", feature = "verify"))] /// Variants of GroveDB stored entities /// @@ -85,6 +98,13 @@ pub enum Element { /// Same as Element::Tree but underlying Merk sums value of it's summable /// nodes SumTree(Option>, SumValue, Option), + /// Same as Element::Tree but underlying Merk sums value of it's summable + /// nodes in big form i128 + /// The big sum tree is valuable if you have a big sum tree of sum trees + BigSumTree(Option>, BigSumValue, Option), + // /// Same as Element::Tree but underlying Merk counts value of its countable + // /// nodes + // CountTree(Option>, CountValue, Option), } impl fmt::Display for Element { @@ -142,6 +162,17 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::BigSumTree(root_key, sum_value, flags) => { + write!( + f, + "BigSumTree({}, {}{})", + root_key.as_ref().map_or("None".to_string(), hex::encode), + sum_value, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } } } } @@ -154,6 +185,7 @@ impl Element { Element::Tree(..) => "tree", Element::SumItem(..) => "sum item", Element::SumTree(..) => "sum tree", + Element::BigSumTree(..) => "big sum tree", } } diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 74cfe8073..3bf1dbe82 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -20,7 +20,7 @@ use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; - +use grovedb_merk::merk::TreeType; use crate::{ batch::{key_info::KeyInfo, KeyInfoPath}, element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, @@ -203,7 +203,7 @@ impl GroveDb { pub fn average_case_merk_insert_element( key: &KeyInfo, value: &Element, - in_tree_using_sums: bool, + in_tree_type: TreeType, propagate_for_level: Option<&EstimatedLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -230,13 +230,13 @@ impl GroveDb { TREE_COST_SIZE }; let value_len = tree_cost_size + flags_len; - add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_tree_using_sums) + add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_tree_type) } _ => add_cost_case_merk_insert( &mut cost, key_len, cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, - in_tree_using_sums, + in_tree_type, ), }; if let Some(level) = propagate_for_level { diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index dce0e1b48..132f0dbb6 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -209,7 +209,8 @@ pub use query::{PathQuery, SizedQuery}; use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] use tokio::net::ToSocketAddrs; - +use grovedb_merk::merk::TreeType; +use grovedb_merk::tree::AggregateData; #[cfg(feature = "full")] use crate::element::helpers::raw_decode; #[cfg(any(feature = "full", feature = "verify"))] @@ -256,7 +257,7 @@ type VerificationIssues = HashMap>, (CryptoHash, CryptoHash, CryptoH type OpenedMerkForReplication<'tx> = ( Merk>, Option>, - bool, + TreeType, ); #[cfg(feature = "full")] @@ -353,7 +354,7 @@ impl GroveDb { &'db self, prefix: SubtreePrefix, root_key: Option>, - is_sum_tree: bool, + tree_type: TreeType, tx: &'db Transaction, batch: Option<&'db StorageBatch>, grove_version: &GroveVersion, @@ -367,7 +368,7 @@ impl GroveDb { Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -421,13 +422,14 @@ impl GroveDb { )) }) .unwrap()?; - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { + let tree_type = element.tree_type(); + if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) | Element::BigSumTree(root_key, ..) = element { + let tree_type = tree_type.expect("expected tree type"); Ok(( Merk::open_layered_with_root_key( storage, root_key.clone(), - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -436,7 +438,7 @@ impl GroveDb { }) .unwrap()?, root_key, - is_sum_tree, + tree_type, )) } else { Err(Error::CorruptedPath( @@ -447,14 +449,14 @@ impl GroveDb { Ok(( Merk::open_base( storage, - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) .unwrap()?, None, - false, + TreeType::NormalTree, )) } } @@ -494,12 +496,13 @@ impl GroveDb { } ) ); - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { + let tree_type = element.tree_type(); + if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) | Element::BigSumTree(root_key, ..) = element { + let tree_type = tree_type.expect("expected tree type"); Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -516,7 +519,7 @@ impl GroveDb { } else { Merk::open_base( storage, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -529,7 +532,7 @@ impl GroveDb { &'db self, prefix: SubtreePrefix, root_key: Option>, - is_sum_tree: bool, + tree_type: TreeType, batch: Option<&'db StorageBatch>, grove_version: &GroveVersion, ) -> CostResult>, Error> { @@ -542,7 +545,7 @@ impl GroveDb { Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -555,7 +558,7 @@ impl GroveDb { } else { Merk::open_base( storage, - false, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -651,7 +654,7 @@ impl GroveDb { grove_version, ) ); - let (root_hash, root_key, sum) = cost_return_on_error!( + let (root_hash, root_key, aggregate_data) = cost_return_on_error!( &mut cost, child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) ); @@ -662,7 +665,7 @@ impl GroveDb { parent_key, root_key, root_hash, - sum, + aggregate_data, grove_version, ) ); @@ -783,7 +786,7 @@ impl GroveDb { key: K, maybe_root_key: Option>, root_tree_hash: Hash, - sum: Option, + aggregate_data: AggregateData, grove_version: &GroveVersion, ) -> CostResult<(), Error> { let key_ref = key.as_ref(); @@ -795,7 +798,20 @@ impl GroveDb { } else if let Element::SumTree(.., flag) = element { let tree = Element::new_sum_tree_with_flags_and_sum_value( maybe_root_key, - sum.unwrap_or_default(), + aggregate_data.as_i64(), + flag, + ); + tree.insert_subtree( + parent_tree, + key.as_ref(), + root_tree_hash, + None, + grove_version, + ) + } else if let Element::BigSumTree(.., flag) = element { + let tree = Element::new_big_sum_tree_with_flags_and_sum_value( + maybe_root_key, + aggregate_data.as_i128(), flag, ); tree.insert_subtree( @@ -825,7 +841,7 @@ impl GroveDb { key: K, maybe_root_key: Option>, root_tree_hash: Hash, - sum: Option, + aggregate_data: AggregateData, batch_operations: &mut Vec>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -836,7 +852,7 @@ impl GroveDb { let tree = Element::new_tree_with_flags(maybe_root_key, flag); let merk_feature_type = cost_return_on_error!( &mut cost, - tree.get_feature_type(parent_tree.is_sum_tree) + tree.get_feature_type(parent_tree.tree_type) .wrap_with_cost(OperationCost::default()) ); tree.insert_subtree_into_batch_operations( @@ -850,12 +866,31 @@ impl GroveDb { } else if let Element::SumTree(.., flag) = element { let tree = Element::new_sum_tree_with_flags_and_sum_value( maybe_root_key, - sum.unwrap_or_default(), + aggregate_data.as_i64(), + flag, + ); + let merk_feature_type = cost_return_on_error!( + &mut cost, + tree.get_feature_type(parent_tree.tree_type) + .wrap_with_cost(OperationCost::default()) + ); + tree.insert_subtree_into_batch_operations( + key, + root_tree_hash, + true, + batch_operations, + merk_feature_type, + grove_version, + ) + } else if let Element::BigSumTree(.., flag) = element { + let tree = Element::new_big_sum_tree_with_flags_and_sum_value( + maybe_root_key, + aggregate_data.as_i128(), flag, ); let merk_feature_type = cost_return_on_error!( &mut cost, - tree.get_feature_type(parent_tree.is_sum_tree) + tree.get_feature_type(parent_tree.tree_type) .wrap_with_cost(OperationCost::default()) ); tree.insert_subtree_into_batch_operations( diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 9244c60be..461d3cf2c 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -711,7 +711,7 @@ impl GroveDb { grove_version ) ); - let uses_sum_tree = subtree_to_delete_from.is_sum_tree; + let uses_sum_tree = subtree_to_delete_from.tree_type; if element.is_any_tree() { let subtree_merk_path = path.derive_owned_with_child(key); let subtree_merk_path_ref = SubtreePath::from(&subtree_merk_path); @@ -905,7 +905,7 @@ impl GroveDb { &mut cost, self.open_non_transactional_merk_at_path(path.clone(), Some(batch), grove_version) ); - let uses_sum_tree = subtree_to_delete_from.is_sum_tree; + let uses_sum_tree = subtree_to_delete_from.tree_type; if element.is_any_tree() { let subtree_merk_path = path.derive_owned_with_child(key); let subtree_of_tree_we_are_deleting = cost_return_on_error!( diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index b255f6535..d72c26f76 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -5,6 +5,7 @@ use grovedb_merk::{ tree::kv::ValueDefinedCostType, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; +use grovedb_merk::tree::AggregateData; use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; @@ -318,7 +319,7 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("node should exist"), Some(SummedMerkNode(0)) )); - assert_eq!(merk.sum().expect("expected to get sum"), Some(40)); + assert_eq!(merk.aggregate_data().expect("expected to get sum").as_i64(), 40); // Perform the same test on regular trees let db = make_test_grovedb(grove_version); @@ -383,7 +384,7 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("node should exist"), Some(BasicMerkNode) )); - assert_eq!(merk.sum().expect("expected to get sum"), None); + assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::NoAggregateData); } #[test] @@ -413,7 +414,7 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.sum().expect("expected to get sum"), None); + assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::NoAggregateData); // Add sum tree db.insert( @@ -452,7 +453,7 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.sum().expect("expected to get sum"), Some(30)); + assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(30)); // Add more sum items db.insert( @@ -483,7 +484,7 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.sum().expect("expected to get sum"), Some(70)); // 30 - 10 + 50 = 70 + assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(70)); // 30 - 10 + 50 = 70 // Add non sum items, result should remain the same db.insert( @@ -504,7 +505,7 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.sum().expect("expected to get sum"), Some(70)); + assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(70)); // Update existing sum items db.insert( @@ -535,7 +536,7 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.sum().expect("expected to get sum"), Some(-60)); // 30 + 10 - 100 = -60 + assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(-60)); // 30 + 10 - 100 = -60 // We can not replace a normal item with a sum item, so let's delete it first db.delete( @@ -566,7 +567,7 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.sum().expect("expected to get sum"), Some(9999940)); // 30 + + assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(9999940)); // 30 + // 10 - // 100 + // 10000000 @@ -865,7 +866,7 @@ fn test_sum_tree_with_batches() { .expect("node should exist"), Some(SummedMerkNode(10)) )); - assert_eq!(sum_tree.sum().expect("expected to get sum"), Some(20)); + assert_eq!(sum_tree.aggregate_data().expect("expected to get sum"), AggregateData::Sum(20)); // Test propagation // Add a new sum tree with its own sum items, should affect sum of original @@ -940,5 +941,5 @@ fn test_sum_tree_with_batches() { ) .unwrap() .expect("should open tree"); - assert_eq!(sum_tree.sum().expect("expected to get sum"), Some(41)); + assert_eq!(sum_tree.aggregate_data().expect("expected to get sum"), AggregateData::Sum(41)); } diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index fc08ab45d..89f7b94b9 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -12,6 +12,7 @@ use crate::{ tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, HASH_LENGTH_U32, }; +use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] /// Average key size @@ -234,8 +235,8 @@ pub type EstimatedToBeEmpty = bool; #[derive(Clone, Copy, PartialEq, Eq, Debug)] /// Information on an estimated layer pub struct EstimatedLayerInformation { - /// Is sum tree? - pub is_sum_tree: bool, + /// The kind of tree we are in + pub tree_type: TreeType, /// Estimated layer count pub estimated_layer_count: EstimatedLayerCount, /// Estimated layer sizes @@ -291,13 +292,13 @@ impl TreeNode { pub fn average_case_encoded_tree_size( not_prefixed_key_len: u32, estimated_element_size: u32, - is_sum_node: bool, + node_type: NodeType, ) -> u32 { // two option values for the left and right link // the actual left and right link encoding size // the encoded kv node size - 2 + (2 * Link::encoded_link_size(not_prefixed_key_len, is_sum_node)) - + KV::encoded_kv_node_size(estimated_element_size, is_sum_node) + 2 + (2 * Link::encoded_link_size(not_prefixed_key_len, node_type)) + + KV::encoded_kv_node_size(estimated_element_size, node_type) } } @@ -307,7 +308,7 @@ pub fn add_average_case_get_merk_node( cost: &mut OperationCost, not_prefixed_key_len: u32, approximate_element_size: u32, - is_sum_tree: bool, + node_type: NodeType, ) -> Result<(), Error> { // Worst case scenario, the element is not already in memory. // One direct seek has to be performed to read the node from storage. @@ -318,7 +319,7 @@ pub fn add_average_case_get_merk_node( cost.storage_loaded_bytes += TreeNode::average_case_encoded_tree_size( not_prefixed_key_len, approximate_element_size, - is_sum_tree, + node_type, ) as u64; Ok(()) } @@ -340,11 +341,11 @@ pub fn add_average_case_merk_replace_layered( cost: &mut OperationCost, key_len: u32, value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) { cost.seek_count += 1; cost.storage_cost.replaced_bytes = - KV::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_len, is_sum_node); + KV::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_len, node_type); // first lets add the value hash cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); @@ -408,12 +409,11 @@ pub fn add_average_case_merk_propagate( let mut nodes_updated = 0; // Propagation requires to recompute and write hashes up to the root let EstimatedLayerInformation { - is_sum_tree, + tree_type, estimated_layer_count, estimated_layer_sizes, } = input; let levels = estimated_layer_count.estimate_levels(); - let in_sum_tree = *is_sum_tree; nodes_updated += levels; if levels > 1 { @@ -442,7 +442,7 @@ pub fn add_average_case_merk_propagate( * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len, - *is_sum_tree, + tree_type.inner_node_type(), ) + sum_tree_addition) } EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) @@ -457,7 +457,7 @@ pub fn add_average_case_merk_propagate( * KV::value_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, average_value_len, - in_sum_tree, + tree_type.inner_node_type(), ) } EstimatedLayerSizes::Mix { @@ -492,7 +492,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len, - in_sum_tree, + tree_type.inner_node_type(), ) + sum_tree_addition; (*weight as u64) .checked_mul(cost as u64) @@ -507,7 +507,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, value_len, - in_sum_tree, + tree_type.inner_node_type(), ); (*weight as u64) .checked_mul(cost as u64) @@ -522,7 +522,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, value_len, - in_sum_tree, + tree_type.inner_node_type(), ); (*weight as u64) .checked_mul(cost as u64) @@ -557,7 +557,7 @@ pub fn add_average_case_merk_propagate( * KV::layered_node_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len + sum_tree_addition, - in_sum_tree, + tree_type.inner_node_type(), ) } EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) @@ -572,7 +572,7 @@ pub fn add_average_case_merk_propagate( * KV::node_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, average_value_len, - in_sum_tree, + tree_type.inner_node_type(), ) } EstimatedLayerSizes::Mix { @@ -608,7 +608,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len + sum_tree_addition, - in_sum_tree, + tree_type.inner_node_type(), ); (*weight as u64) .checked_mul(cost as u64) @@ -625,7 +625,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, value_len, - in_sum_tree, + tree_type.inner_node_type(), ); (*weight as u64) .checked_mul(cost as u64) @@ -642,7 +642,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, value_len, - false, + tree_type.inner_node_type(), //todo this is an error that we will need to fix however most likely references were never in sum trees ); (*weight as u64) .checked_mul(cost as u64) diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index bd669db12..6d0f47a67 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -7,6 +7,7 @@ use integer_encoding::VarInt; #[cfg(feature = "full")] use crate::{tree::kv::KV, HASH_BLOCK_SIZE_U32, HASH_LENGTH_U32}; +use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] pub mod average_case_costs; @@ -26,17 +27,26 @@ pub const LAYER_COST_SIZE: u32 = 3; /// The cost of a sum value pub const SUM_VALUE_EXTRA_COST: u32 = 9; +#[cfg(any(feature = "full", feature = "verify"))] +/// The cost of a big sum value +pub const BIG_SUM_VALUE_EXTRA_COST: u32 = 16; + #[cfg(feature = "full")] /// The cost of a summed subtree layer /// This is the layer size + 9 for the encoded value pub const SUM_LAYER_COST_SIZE: u32 = LAYER_COST_SIZE + SUM_VALUE_EXTRA_COST; +#[cfg(feature = "full")] +/// The cost of a summed subtree layer +/// This is the layer size + 16 for the encoded value +pub const BIG_SUM_LAYER_COST_SIZE: u32 = LAYER_COST_SIZE + BIG_SUM_VALUE_EXTRA_COST; + #[cfg(feature = "full")] impl KV { - fn encoded_kv_node_size(element_size: u32, is_sum_node: bool) -> u32 { + fn encoded_kv_node_size(element_size: u32, node_type: NodeType) -> u32 { // We always charge 8 bytes for the sum node (even though // it could theoretically be 9 bytes - let sum_node_feature_size = if is_sum_node { 9 } else { 1 }; + let sum_node_feature_size = node_type.feature_len(); // KV holds the state of a node // 32 bytes to encode the hash of the node // 32 bytes to encode the value hash @@ -51,13 +61,13 @@ pub fn add_cost_case_merk_insert( cost: &mut OperationCost, key_len: u32, value_len: u32, - in_tree_using_sums: bool, + in_tree_type: TreeType, ) { cost.seek_count += 1; cost.storage_cost.added_bytes += KV::node_byte_cost_size_for_key_and_raw_value_lengths( key_len, value_len, - in_tree_using_sums, + in_tree_type.inner_node_type(), ); // .. and hash computation for the inserted element itself // first lets add the value hash @@ -75,13 +85,13 @@ pub fn add_cost_case_merk_insert_layered( cost: &mut OperationCost, key_len: u32, value_len: u32, - in_tree_using_sums: bool, + in_tree_type: TreeType, ) { cost.seek_count += 1; cost.storage_cost.added_bytes += KV::layered_node_byte_cost_size_for_key_and_value_lengths( key_len, value_len, - in_tree_using_sums, + in_tree_type.inner_node_type(), ); // .. and hash computation for the inserted element itself // first lets add the value hash @@ -101,11 +111,11 @@ pub fn add_cost_case_merk_replace( cost: &mut OperationCost, key_len: u32, value_len: u32, - in_tree_using_sums: bool, + in_tree_type: TreeType, ) { cost.seek_count += 1; cost.storage_cost.added_bytes += - KV::node_value_byte_cost_size(key_len, value_len, in_tree_using_sums); + KV::node_value_byte_cost_size(key_len, value_len, in_tree_type.inner_node_type()); cost.storage_cost.replaced_bytes += KV::node_key_byte_cost_size(key_len); // .. and hash computation for the inserted element itself // first lets add the value hash @@ -124,13 +134,13 @@ pub fn add_cost_case_merk_replace_same_size( cost: &mut OperationCost, key_len: u32, value_len: u32, - in_tree_using_sums: bool, + in_tree_type: TreeType, ) { cost.seek_count += 1; cost.storage_cost.replaced_bytes += KV::node_byte_cost_size_for_key_and_raw_value_lengths( key_len, value_len, - in_tree_using_sums, + in_tree_type.inner_node_type(), ); // .. and hash computation for the inserted element itself // first lets add the value hash @@ -148,13 +158,13 @@ pub fn add_cost_case_merk_replace_layered( cost: &mut OperationCost, key_len: u32, value_len: u32, - in_tree_using_sums: bool, + in_tree_type: TreeType, ) { cost.seek_count += 1; cost.storage_cost.replaced_bytes += KV::layered_node_byte_cost_size_for_key_and_value_lengths( key_len, value_len, - in_tree_using_sums, + in_tree_type.inner_node_type(), ); // .. and hash computation for the inserted element itself // first lets add the value hash @@ -176,7 +186,7 @@ pub fn add_cost_case_merk_patch( key_len: u32, value_len: u32, change_in_bytes: i32, - in_tree_using_sums: bool, + in_tree_type: TreeType, ) { cost.seek_count += 1; if change_in_bytes >= 0 { @@ -185,12 +195,12 @@ pub fn add_cost_case_merk_patch( let old_byte_size = KV::node_byte_cost_size_for_key_and_raw_value_lengths( key_len, value_len - change_in_bytes as u32, - in_tree_using_sums, + in_tree_type.inner_node_type(), ); let new_byte_size = KV::node_byte_cost_size_for_key_and_raw_value_lengths( key_len, value_len, - in_tree_using_sums, + in_tree_type.inner_node_type(), ); cost.storage_cost.replaced_bytes += old_byte_size; @@ -199,7 +209,7 @@ pub fn add_cost_case_merk_patch( cost.storage_cost.replaced_bytes += KV::node_byte_cost_size_for_key_and_raw_value_lengths( key_len, value_len, - in_tree_using_sums, + in_tree_type.inner_node_type(), ); } diff --git a/merk/src/estimated_costs/worst_case_costs.rs b/merk/src/estimated_costs/worst_case_costs.rs index 9ae6c2b3c..0f206700a 100644 --- a/merk/src/estimated_costs/worst_case_costs.rs +++ b/merk/src/estimated_costs/worst_case_costs.rs @@ -40,6 +40,7 @@ use crate::{ tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, }; +use crate::merk::NodeType; #[cfg(feature = "full")] #[derive(Clone, PartialEq, Eq, Debug)] @@ -57,13 +58,13 @@ impl TreeNode { pub fn worst_case_encoded_tree_size( not_prefixed_key_len: u32, max_element_size: u32, - is_sum_node: bool, + node_type: NodeType, ) -> u32 { // two option values for the left and right link // the actual left and right link encoding size // the encoded kv node size - 2 + (2 * Link::encoded_link_size(not_prefixed_key_len, is_sum_node)) - + KV::encoded_kv_node_size(max_element_size, is_sum_node) + 2 + (2 * Link::encoded_link_size(not_prefixed_key_len, node_type)) + + KV::encoded_kv_node_size(max_element_size, node_type) } } @@ -73,7 +74,7 @@ pub fn add_worst_case_get_merk_node( cost: &mut OperationCost, not_prefixed_key_len: u32, max_element_size: u32, - is_sum_node: bool, + node_type: NodeType, ) -> Result<(), Error> { // Worst case scenario, the element is not already in memory. // One direct seek has to be performed to read the node from storage. @@ -82,7 +83,7 @@ pub fn add_worst_case_get_merk_node( // To write a node to disk, the left link, right link and kv nodes are encoded. // worst case, the node has both the left and right link present. cost.storage_loaded_bytes += - TreeNode::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, is_sum_node) + TreeNode::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, node_type) as u64; Ok(()) } @@ -104,10 +105,10 @@ pub fn add_worst_case_merk_insert( cost: &mut OperationCost, key_len: u32, value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) { cost.storage_cost.added_bytes += - KV::node_byte_cost_size_for_key_and_raw_value_lengths(key_len, value_len, is_sum_node); + KV::node_byte_cost_size_for_key_and_raw_value_lengths(key_len, value_len, node_type); // .. and hash computation for the inserted element itself // todo: verify this cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); @@ -119,12 +120,12 @@ pub fn add_worst_case_merk_replace_layered( cost: &mut OperationCost, key_len: u32, value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) { // todo: verify this cost.hash_node_calls += 1 + ((value_len - 1) / HASH_BLOCK_SIZE_U32); cost.storage_cost.replaced_bytes = - KV::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_len, is_sum_node); + KV::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_len, node_type); // 37 + 35 + key_len } diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 9c5c9ec9f..795124926 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -17,6 +17,7 @@ use crate::{ }, Error, Merk, MerkBatch, MerkOptions, }; +use crate::merk::NodeType; impl<'db, S> Merk where @@ -64,7 +65,7 @@ where KB: AsRef<[u8]>, KA: AsRef<[u8]>, { - let use_sum_nodes = self.is_sum_tree; + let node_type : NodeType = self.tree_type.inner_node_type(); self.apply_with_costs_just_in_time_value_update( batch, aux, @@ -73,7 +74,7 @@ where Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( key.len() as u32, value.len() as u32, - use_sum_nodes, + node_type, )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index b7b181caa..e1dfdccd0 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -243,6 +243,53 @@ impl MerkType { } } } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum TreeType { + NormalTree, + SumTree, + BigSumTree, + CountTree, +} + +impl TreeType { + pub fn inner_node_type(&self) -> NodeType { + match self { + TreeType::NormalTree => NodeType::NormalNode, + TreeType::SumTree => NodeType::SumNode, + TreeType::BigSumTree => NodeType::BigSumNode, + TreeType::CountTree => NodeType::CountNode, + } + } +} + +#[cfg(any(feature = "full", feature = "verify"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum NodeType { + NormalNode, + SumNode, + BigSumNode, + CountNode, +} + +impl NodeType { + pub const fn feature_len(&self) -> u32 { + match self { + NodeType::NormalNode => 1, + NodeType::SumNode => 9, + NodeType::BigSumNode => 17, + NodeType::CountNode => 9, + } + } + + pub const fn cost(&self) -> u32 { + match self { + NodeType::NormalNode => 0, + NodeType::SumNode => 8, + NodeType::BigSumNode => 16, + NodeType::CountNode => 8, + } + } +} /// A handle to a Merkle key/value store backed by RocksDB. pub struct Merk { @@ -252,8 +299,8 @@ pub struct Merk { pub storage: S, /// Merk type pub merk_type: MerkType, - /// Is sum tree? - pub is_sum_tree: bool, + /// The tree type + pub tree_type: TreeType, } impl fmt::Debug for Merk { diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index c8646afaf..d47febcd4 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -9,26 +9,27 @@ use crate::{ Error, Merk, MerkType, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, }; +use crate::merk::TreeType; impl<'db, S> Merk where S: StorageContext<'db>, { /// Open empty tree - pub fn open_empty(storage: S, merk_type: MerkType, is_sum_tree: bool) -> Self { + pub fn open_empty(storage: S, merk_type: MerkType, tree_type: TreeType) -> Self { Self { tree: Cell::new(None), root_tree_key: Cell::new(None), storage, merk_type, - is_sum_tree, + tree_type, } } /// Open standalone tree pub fn open_standalone( storage: S, - is_sum_tree: bool, + tree_type: TreeType, value_defined_cost_fn: Option< impl Fn(&[u8], &GroveVersion) -> Option, >, @@ -39,7 +40,7 @@ where root_tree_key: Cell::new(None), storage, merk_type: StandaloneMerk, - is_sum_tree, + tree_type, }; merk.load_base_root(value_defined_cost_fn, grove_version) @@ -49,7 +50,7 @@ where /// Open base tree pub fn open_base( storage: S, - is_sum_tree: bool, + tree_type: TreeType, value_defined_cost_fn: Option< impl Fn(&[u8], &GroveVersion) -> Option, >, @@ -60,7 +61,7 @@ where root_tree_key: Cell::new(None), storage, merk_type: BaseMerk, - is_sum_tree, + tree_type, }; merk.load_base_root(value_defined_cost_fn, grove_version) @@ -71,7 +72,7 @@ where pub fn open_layered_with_root_key( storage: S, root_key: Option>, - is_sum_tree: bool, + tree_type: TreeType, value_defined_cost_fn: Option< impl Fn(&[u8], &GroveVersion) -> Option, >, @@ -82,7 +83,7 @@ where root_tree_key: Cell::new(root_key), storage, merk_type: LayeredMerk, - is_sum_tree, + tree_type, }; merk.load_root(value_defined_cost_fn, grove_version) @@ -102,6 +103,7 @@ mod test { use tempfile::TempDir; use crate::{tree::kv::ValueDefinedCostType, Merk, Op, TreeFeatureType::BasicMerkNode}; + use crate::merk::TreeType; #[test] fn test_reopen_root_hash() { @@ -116,7 +118,7 @@ mod test { storage .get_storage_context(SubtreePath::from(test_prefix.as_ref()), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -143,7 +145,7 @@ mod test { storage .get_storage_context(SubtreePath::from(test_prefix.as_ref()), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -162,7 +164,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ); @@ -192,7 +194,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ); diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index 55ee663ec..cb694b418 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -52,6 +52,7 @@ use crate::{ Error::{CostsError, StorageError}, Link, Merk, }; +use crate::merk::TreeType; /// Restorer handles verification of chunks and replication of Merk trees. /// Chunks can be processed randomly as long as their parent has been processed @@ -449,7 +450,7 @@ impl<'db, S: StorageContext<'db>> Restorer { if !self .merk - .verify(self.merk.is_sum_tree, grove_version) + .verify(self.merk.tree_type == TreeType::NormalTree, grove_version) .0 .is_empty() { @@ -564,6 +565,7 @@ mod tests { Error::ChunkRestoringError, Merk, PanicSource, }; + use crate::merk::TreeType; #[test] fn test_chunk_verification_non_avl_tree() { @@ -682,7 +684,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -932,7 +934,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -951,7 +953,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1024,7 +1026,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1093,7 +1095,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1175,7 +1177,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1277,7 +1279,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1359,7 +1361,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1409,7 +1411,7 @@ mod tests { storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs index dd71e74ed..620373a86 100644 --- a/merk/src/merk/source.rs +++ b/merk/src/merk/source.rs @@ -6,6 +6,7 @@ use crate::{ tree::{kv::ValueDefinedCostType, Fetch, TreeNode}, Error, Link, Merk, }; +use crate::merk::TreeType; impl<'db, S> Merk where @@ -14,7 +15,7 @@ where pub(in crate::merk) fn source(&self) -> MerkSource { MerkSource { storage: &self.storage, - is_sum_tree: self.is_sum_tree, + tree_type: self.tree_type, } } } @@ -22,14 +23,14 @@ where #[derive(Debug)] pub struct MerkSource<'s, S> { storage: &'s S, - is_sum_tree: bool, + tree_type: TreeType, } impl<'s, S> Clone for MerkSource<'s, S> { fn clone(&self) -> Self { MerkSource { storage: self.storage, - is_sum_tree: self.is_sum_tree, + tree_type: self.tree_type, } } } diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 45beda4fe..24ac94fad 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -47,6 +47,7 @@ use crate::{ Merk, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; +use crate::merk::TreeType; /// Assert tree invariants pub fn assert_tree_invariants(tree: &TreeNode) { @@ -80,7 +81,7 @@ pub fn apply_memonly_unchecked( batch: &MerkBatch>, grove_version: &GroveVersion, ) -> TreeNode { - let is_sum_node = tree.is_sum_node(); + let node_type = tree.node_type(); let walker = Walker::::new(tree, PanicSource {}); let mut tree = Walker::::apply_to( Some(walker), @@ -90,7 +91,7 @@ pub fn apply_memonly_unchecked( Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( key.len() as u32, value.len() as u32, - is_sum_node, + node_type, )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -108,12 +109,12 @@ pub fn apply_memonly_unchecked( .expect("apply failed") .0 .expect("expected tree"); - let is_sum_node = tree.is_sum_node(); + let node_type = tree.node_type(); tree.commit(&mut NoopCommit {}, &|key, value| { Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( key.len() as u32, value.len() as u32, - is_sum_node, + node_type, )) }) .unwrap() @@ -138,7 +139,7 @@ pub fn apply_memonly( pub fn apply_to_memonly( maybe_tree: Option, batch: &MerkBatch>, - is_sum_tree: bool, + tree_type: TreeType, grove_version: &GroveVersion, ) -> Option { let maybe_walker = maybe_tree.map(|tree| Walker::::new(tree, PanicSource {})); @@ -150,7 +151,7 @@ pub fn apply_to_memonly( Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( key.len() as u32, value.len() as u32, - is_sum_tree, + tree_type.inner_node_type(), )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -168,12 +169,12 @@ pub fn apply_to_memonly( .expect("apply failed") .0 .map(|mut tree| { - let is_sum_node = tree.is_sum_node(); + let node_type = tree.node_type(); tree.commit(&mut NoopCommit {}, &|key, value| { Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( key.len() as u32, value.len() as u32, - is_sum_node, + node_type, )) }) .unwrap() @@ -320,7 +321,7 @@ where storage .get_storage_context(SubtreePath::empty(), Some(batch)) .unwrap(), - false, + TreeType::NormalTree, None:: Option>, grove_version, ) @@ -340,7 +341,7 @@ where storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None:: Option>, grove_version, ) diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 69e5b5550..3133ce6bb 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -43,6 +43,7 @@ use grovedb_version::version::GroveVersion; use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::Merk; +use crate::merk::TreeType; #[cfg(feature = "full")] /// Wraps a Merk instance and deletes it from disk it once it goes out of scope. @@ -66,7 +67,7 @@ impl TempMerk { let merk = Merk::open_base( context, - false, + TreeType::NormalTree, None:: Option>, grove_version, ) @@ -93,7 +94,7 @@ impl TempMerk { .unwrap(); self.merk = Merk::open_base( context, - false, + TreeType::NormalTree, None:: Option>, grove_version, ) diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index ab8f50718..31ef980e9 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -146,6 +146,7 @@ impl TreeNode { #[cfg(feature = "full")] #[cfg(test)] mod tests { + use crate::tree::AggregateData; use super::{super::Link, *}; use crate::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; @@ -196,7 +197,7 @@ mod tests { [55; 32], Some(Link::Loaded { hash: [66; 32], - sum: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (123, 124), tree: TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap(), }), @@ -225,7 +226,7 @@ mod tests { [55; 32], Some(Link::Uncommitted { hash: [66; 32], - sum: Some(10), + aggregate_data: AggregateData::Sum(10), child_heights: (123, 124), tree: TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap(), }), @@ -254,7 +255,7 @@ mod tests { [55; 32], Some(Link::Reference { hash: [66; 32], - aggregate_data: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (123, 124), key: vec![2], }), diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index f4a0e2248..ccad4dd2a 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -21,7 +21,7 @@ use crate::{ }, Link, HASH_LENGTH_U32, HASH_LENGTH_U32_X2, }; - +use crate::merk::NodeType; // TODO: maybe use something similar to Vec but without capacity field, // (should save 16 bytes per entry). also, maybe a shorter length // field to save even more. also might be possible to combine key @@ -275,16 +275,16 @@ impl KV { pub fn node_value_byte_cost_size( not_prefixed_key_len: u32, raw_value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) -> u32 { // Sum trees are either 1 or 9 bytes. While they might be more or less on disk, // costs can not take advantage of the varint aspect of the feature. - let feature_len = if is_sum_node { 9 } else { 1 }; + let feature_len = node_type.feature_len(); let value_size = raw_value_len + HASH_LENGTH_U32_X2 + feature_len; // The node will be a child of another node which stores it's key and hash // That will be added during propagation - let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, is_sum_node); + let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, node_type); value_size + value_size.required_space() as u32 + parent_to_child_cost } @@ -294,10 +294,10 @@ impl KV { pub fn node_byte_cost_size_for_key_and_raw_value_lengths( not_prefixed_key_len: u32, raw_value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) -> u32 { let node_value_size = - Self::node_value_byte_cost_size(not_prefixed_key_len, raw_value_len, is_sum_node); + Self::node_value_byte_cost_size(not_prefixed_key_len, raw_value_len, node_type); let node_key_size = Self::node_key_byte_cost_size(not_prefixed_key_len); // Each node stores the key and value, the value hash and node hash node_value_size + node_key_size @@ -308,11 +308,11 @@ impl KV { pub fn layered_node_byte_cost_size_for_key_and_value_lengths( not_prefixed_key_len: u32, value_len: u32, - is_sum_node: bool, // this means the node is contained in a sumtree + node_type: NodeType, ) -> u32 { // Sum trees are either 1 or 9 bytes. While they might be more or less on disk, // costs can not take advantage of the varint aspect of the feature. - let feature_len = if is_sum_node { 9 } else { 1 }; + let feature_len = node_type.feature_len(); // Each node stores the key and value, and the node hash // the value hash on a layered node is not stored directly in the node @@ -326,7 +326,7 @@ impl KV { let node_size = node_value_size + node_key_size; // The node will be a child of another node which stores it's key and hash // That will be added during propagation - let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, is_sum_node); + let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, node_type); node_size + parent_to_child_cost } @@ -336,11 +336,12 @@ impl KV { pub fn layered_value_byte_cost_size_for_key_and_value_lengths( not_prefixed_key_len: u32, value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) -> u32 { - // Sum trees are either 1 or 9 bytes. While they might be more or less on disk, + // Sum trees are either 1 or 9 bytes, or 16 bytes for the big sum trees. + // While they might be more or less on disk, // costs can not take advantage of the varint aspect of the feature. - let feature_len = if is_sum_node { 9 } else { 1 }; + let feature_len = node_type.feature_len(); // Each node stores the key and value, and the node hash // the value hash on a layered node is not stored directly in the node // The required space is set to 2. However in reality it could be 1 or 2. @@ -352,7 +353,7 @@ impl KV { let node_value_size = value_len + feature_len + HASH_LENGTH_U32 + 2; // The node will be a child of another node which stores it's key and hash // That will be added during propagation - let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, is_sum_node); + let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, node_type); node_value_size + parent_to_child_cost } @@ -362,7 +363,7 @@ impl KV { pub fn value_byte_cost_size_for_key_and_value_lengths( not_prefixed_key_len: u32, value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) -> u32 { // encoding a reference encodes the key last and doesn't encode the size of the // key. so no need for a varint required space calculation for the @@ -371,7 +372,7 @@ impl KV { // however we do need the varint required space for the cost of the key in // rocks_db let parent_to_child_reference_len = - Link::encoded_link_size(not_prefixed_key_len, is_sum_node); + Link::encoded_link_size(not_prefixed_key_len, node_type); value_len + value_len.required_space() as u32 + parent_to_child_reference_len } @@ -381,14 +382,14 @@ impl KV { pub(crate) fn value_byte_cost_size_for_key_and_raw_value_lengths( not_prefixed_key_len: u32, raw_value_len: u32, - is_sum_node: bool, + node_type: NodeType, ) -> u32 { - let sum_tree_len = if is_sum_node { 9 } else { 1 }; // 1 for option, 0 or 9 for sum feature + let sum_tree_len = node_type.feature_len(); // 1 for option, 0 or 9 for sum feature let value_len = raw_value_len + HASH_LENGTH_U32_X2 + sum_tree_len; Self::value_byte_cost_size_for_key_and_value_lengths( not_prefixed_key_len, value_len, - is_sum_node, + node_type, ) } @@ -400,7 +401,7 @@ impl KV { Self::value_byte_cost_size_for_key_and_value_lengths( key_len, value_len, - self.feature_type.is_sum_feature(), + self.feature_type.node_type(), ) } @@ -415,12 +416,12 @@ impl KV { #[inline] pub(crate) fn layered_value_byte_cost_size(&self, value_cost: u32) -> u32 { let key_len = self.key.len() as u32; - let is_sum_node = self.feature_type.is_sum_feature(); + let node_type = self.feature_type.node_type(); Self::layered_value_byte_cost_size_for_key_and_value_lengths( key_len, value_cost, - is_sum_node, + node_type, ) } @@ -431,9 +432,9 @@ impl KV { #[inline] pub(crate) fn specialized_value_byte_cost_size(&self, value_cost: u32) -> u32 { let key_len = self.key.len() as u32; - let is_sum_node = self.feature_type.is_sum_feature(); + let node_type = self.feature_type.node_type(); - Self::node_value_byte_cost_size(key_len, value_cost, is_sum_node) + Self::node_value_byte_cost_size(key_len, value_cost, node_type) } /// Costs based on predefined types (Trees, SumTrees, SumItems) that behave diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index 143207548..2537b15bd 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -12,6 +12,7 @@ use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; use super::{hash::CryptoHash, TreeNode}; #[cfg(feature = "full")] use crate::HASH_LENGTH_U32; +use crate::merk::NodeType; #[cfg(feature = "full")] use crate::tree::tree_feature_type::AggregateData; // TODO: optimize memory footprint @@ -252,8 +253,8 @@ impl Link { // Costs for operations within a single merk #[inline] /// Encoded link size - pub const fn encoded_link_size(not_prefixed_key_len: u32, is_sum_tree: bool) -> u32 { - let sum_tree_cost = if is_sum_tree { 8 } else { 0 }; + pub const fn encoded_link_size(not_prefixed_key_len: u32, node_type: NodeType) -> u32 { + let sum_tree_cost = node_type.cost(); // Links are optional values that represent the right or left node for a given // 1 byte to represent key_length (this is a u8) // key_length to represent the actual key diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 084f3059f..f82f92aa3 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -74,6 +74,7 @@ use crate::tree::kv::ValueDefinedCostType; use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "full")] use crate::{error::Error, Error::Overflow}; +use crate::merk::NodeType; // TODO: remove need for `TreeInner`, and just use `Box` receiver for // relevant methods @@ -157,9 +158,9 @@ impl TreeNode { } } - /// Is sum node? - pub fn is_sum_node(&self) -> bool { - self.inner.kv.feature_type.is_sum_feature() + /// the node type + pub fn node_type(&self) -> NodeType { + self.inner.kv.feature_type.node_type() } pub fn storage_cost_for_update(current_value_byte_cost: u32, old_cost: u32) -> StorageCost { @@ -252,7 +253,7 @@ impl TreeNode { KV::value_byte_cost_size_for_key_and_value_lengths( key_len, value_len as u32, - self.inner.kv.feature_type.is_sum_feature(), + self.inner.kv.feature_type.node_type(), ) } else { self.inner.kv.value_byte_cost_size() diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index e5f0e2763..7dd6472e2 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -2,14 +2,14 @@ #[cfg(any(feature = "full", feature = "verify"))] use std::io::{Read, Write}; -use byteorder::{BigEndian, ByteOrder, LittleEndian, ReadBytesExt, WriteBytesExt}; +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "full")] use ed::Terminated; #[cfg(any(feature = "full", feature = "verify"))] use ed::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; - +use crate::merk::NodeType; #[cfg(any(feature = "full", feature = "verify"))] use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; use crate::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; @@ -28,8 +28,19 @@ pub enum TreeFeatureType { CountedMerkNode(u64), } +impl TreeFeatureType { + pub fn node_type(&self) -> NodeType { + match self { + BasicMerkNode => NodeType::NormalNode, + SummedMerkNode(_) => NodeType::SumNode, + BigSummedMerkNode(_) => NodeType::BigSumNode, + CountedMerkNode(_) => NodeType::CountNode, + } + } +} + #[cfg(any(feature = "full", feature = "verify"))] -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum AggregateData { NoAggregateData, Sum(i64), @@ -37,6 +48,72 @@ pub enum AggregateData { Count(u64), } +impl AggregateData { + pub fn as_i64(&self) -> i64 { + match self { + AggregateData::NoAggregateData => 0, + AggregateData::Sum(s) => *s, + AggregateData::BigSum(i) => { + let max = i64::MAX as i128; + if *i > max { + i64::MAX + } else { + *i as i64 + } + } + AggregateData::Count(c) => { + let max = i64::MAX as u64; + if *c > max { + i64::MAX + } else { + *c as i64 + } + } + } +} + + pub fn as_u64(&self) -> u64 { + match self { + AggregateData::NoAggregateData => 0, + AggregateData::Sum(s) => { + if *s < 0 { + 0 + } else { + *s as u64 + } + }, + AggregateData::BigSum(i) => { + let max = u64::MAX as i128; + if *i > max { + u64::MAX + } else if *i < 0 { + 0 + } else { + *i as u64 + } + } + AggregateData::Count(c) => { + *c + } + } + } + + pub fn as_i128(&self) -> i128 { + match self { + AggregateData::NoAggregateData => 0, + AggregateData::Sum(s) => { + *s as i128 + }, + AggregateData::BigSum(i) => { + *i + } + AggregateData::Count(c) => { + *c as i128 + } + } + } +} + impl From for AggregateData { fn from(value: TreeFeatureType) -> Self { match value { @@ -61,12 +138,6 @@ impl TreeFeatureType { } } - #[inline] - /// Is sum feature? - pub fn is_sum_feature(&self) -> bool { - matches!(self, SummedMerkNode(_)) - } - #[inline] /// Get encoding cost of self pub(crate) fn encoding_cost(&self) -> usize { From a414fa256b007bf106b8d5468caf7476289f679d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 8 Jan 2025 05:50:22 +0700 Subject: [PATCH 03/30] more work --- grovedb-version/src/version/mod.rs | 5 +- grovedb-version/src/version/v2.rs | 188 ++++++++++++++++++ grovedb/src/batch/batch_structure.rs | 11 +- .../batch/estimated_costs/worst_case_costs.rs | 35 ++-- grovedb/src/batch/mod.rs | 143 ++++++------- .../src/batch/single_deletion_cost_tests.rs | 12 +- .../single_sum_item_deletion_cost_tests.rs | 5 +- grovedb/src/element/constructor.rs | 31 ++- grovedb/src/element/delete.rs | 57 ++++-- grovedb/src/element/get.rs | 127 ++++++++++-- grovedb/src/element/helpers.rs | 49 ++++- grovedb/src/element/insert.rs | 14 +- grovedb/src/element/mod.rs | 43 +++- grovedb/src/element/query.rs | 2 + .../src/estimated_costs/worst_case_costs.rs | 95 ++++----- grovedb/src/lib.rs | 13 +- .../src/operations/delete/delete_up_tree.rs | 9 +- grovedb/src/operations/delete/mod.rs | 25 +-- grovedb/src/operations/is_empty_tree.rs | 2 +- grovedb/src/operations/proof/verify.rs | 4 +- grovedb/src/replication.rs | 27 ++- grovedb/src/replication/state_sync_session.rs | 15 +- grovedb/src/tests/mod.rs | 8 +- grovedb/src/util.rs | 108 +++------- grovedb/src/visualize.rs | 22 ++ merk/src/error.rs | 3 + merk/src/merk/mod.rs | 54 ++++- merk/src/tree/mod.rs | 6 + 28 files changed, 737 insertions(+), 376 deletions(-) create mode 100644 grovedb-version/src/version/v2.rs diff --git a/grovedb-version/src/version/mod.rs b/grovedb-version/src/version/mod.rs index 06ac4e120..8f6268419 100644 --- a/grovedb-version/src/version/mod.rs +++ b/grovedb-version/src/version/mod.rs @@ -1,11 +1,12 @@ pub mod grovedb_versions; pub mod merk_versions; pub mod v1; +pub mod v2; pub use versioned_feature_core::*; use crate::version::{ - grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, + grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2 }; #[derive(Clone, Debug, Default)] @@ -23,4 +24,4 @@ impl GroveVersion { } } -pub const GROVE_VERSIONS: &[GroveVersion] = &[GROVE_V1]; +pub const GROVE_VERSIONS: &[GroveVersion] = &[GROVE_V1, GROVE_V2]; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs new file mode 100644 index 000000000..5e8c58e7d --- /dev/null +++ b/grovedb-version/src/version/v2.rs @@ -0,0 +1,188 @@ +use crate::version::{ + grovedb_versions::{ + GroveDBApplyBatchVersions, GroveDBElementMethodVersions, + GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, + GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, + GroveDBOperationsInsertVersions, GroveDBOperationsProofVersions, + GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, + GroveDBReplicationVersions, GroveDBVersions, + }, + merk_versions::MerkVersions, + GroveVersion, +}; + +pub const GROVE_V2: GroveVersion = GroveVersion { + protocol_version: 0, + grovedb_versions: GroveDBVersions { + apply_batch: GroveDBApplyBatchVersions { + apply_batch_structure: 0, + apply_body: 0, + continue_partial_apply_body: 0, + apply_operations_without_batching: 0, + apply_batch: 0, + apply_partial_batch: 0, + open_batch_transactional_merk_at_path: 0, + open_batch_merk_at_path: 0, + apply_batch_with_element_flags_update: 0, + apply_partial_batch_with_element_flags_update: 0, + estimated_case_operations_for_batch: 0, + }, + element: GroveDBElementMethodVersions { + delete: 0, + delete_with_sectioned_removal_bytes: 0, + delete_into_batch_operations: 0, + element_at_key_already_exists: 0, + get: 0, + get_optional: 0, + get_from_storage: 0, + get_optional_from_storage: 1, + get_with_absolute_refs: 0, + get_value_hash: 0, + get_specialized_cost: 0, + value_defined_cost: 0, + value_defined_cost_for_serialized_value: 0, + specialized_costs_for_key_value: 0, + required_item_space: 0, + insert: 0, + insert_into_batch_operations: 0, + insert_if_not_exists: 0, + insert_if_not_exists_into_batch_operations: 0, + insert_if_changed_value: 0, + insert_if_changed_value_into_batch_operations: 0, + insert_reference: 0, + insert_reference_into_batch_operations: 0, + insert_subtree: 0, + insert_subtree_into_batch_operations: 0, + get_query: 0, + get_query_values: 0, + get_query_apply_function: 0, + get_path_query: 0, + get_sized_query: 0, + path_query_push: 0, + query_item: 0, + basic_push: 0, + serialize: 0, + serialized_size: 0, + deserialize: 0, + }, + operations: GroveDBOperationsVersions { + get: GroveDBOperationsGetVersions { + get: 0, + get_caching_optional: 0, + follow_reference: 0, + get_raw: 0, + get_raw_caching_optional: 0, + get_raw_optional: 0, + get_raw_optional_caching_optional: 0, + has_raw: 0, + check_subtree_exists_invalid_path: 0, + average_case_for_has_raw: 0, + average_case_for_has_raw_tree: 0, + average_case_for_get_raw: 0, + average_case_for_get: 0, + average_case_for_get_tree: 0, + worst_case_for_has_raw: 0, + worst_case_for_get_raw: 0, + worst_case_for_get: 0, + is_empty_tree: 0, + }, + insert: GroveDBOperationsInsertVersions { + insert: 0, + insert_on_transaction: 0, + insert_without_transaction: 0, + add_element_on_transaction: 0, + add_element_without_transaction: 0, + insert_if_not_exists: 0, + insert_if_not_exists_return_existing_element: 0, + insert_if_changed_value: 0, + }, + delete: GroveDBOperationsDeleteVersions { + delete: 0, + clear_subtree: 0, + delete_with_sectional_storage_function: 0, + delete_if_empty_tree: 0, + delete_if_empty_tree_with_sectional_storage_function: 0, + delete_operation_for_delete_internal: 0, + delete_internal_on_transaction: 0, + delete_internal_without_transaction: 0, + average_case_delete_operation_for_delete: 0, + worst_case_delete_operation_for_delete: 0, + }, + delete_up_tree: GroveDBOperationsDeleteUpTreeVersions { + delete_up_tree_while_empty: 0, + delete_up_tree_while_empty_with_sectional_storage: 0, + delete_operations_for_delete_up_tree_while_empty: 0, + add_delete_operations_for_delete_up_tree_while_empty: 0, + average_case_delete_operations_for_delete_up_tree_while_empty: 0, + worst_case_delete_operations_for_delete_up_tree_while_empty: 0, + }, + query: GroveDBOperationsQueryVersions { + query_encoded_many: 0, + query_many_raw: 0, + get_proved_path_query: 0, + query: 0, + query_item_value: 0, + query_item_value_or_sum: 0, + query_sums: 0, + query_raw: 0, + query_keys_optional: 0, + query_raw_keys_optional: 0, + follow_element: 0, + }, + proof: GroveDBOperationsProofVersions { + prove_query: 0, + prove_query_many: 0, + verify_query_with_options: 0, + verify_query_raw: 0, + verify_layer_proof: 0, + verify_query: 0, + verify_subset_query: 0, + verify_query_with_absence_proof: 0, + verify_subset_query_with_absence_proof: 0, + verify_query_with_chained_path_queries: 0, + }, + average_case: GroveDBOperationsAverageCaseVersions { + add_average_case_get_merk_at_path: 0, + average_case_merk_replace_tree: 0, + average_case_merk_insert_tree: 0, + average_case_merk_delete_tree: 0, + average_case_merk_insert_element: 0, + average_case_merk_replace_element: 0, + average_case_merk_patch_element: 0, + average_case_merk_delete_element: 0, + add_average_case_has_raw_cost: 0, + add_average_case_has_raw_tree_cost: 0, + add_average_case_get_raw_cost: 0, + add_average_case_get_raw_tree_cost: 0, + add_average_case_get_cost: 0, + }, + worst_case: GroveDBOperationsWorstCaseVersions { + add_worst_case_get_merk_at_path: 0, + worst_case_merk_replace_tree: 0, + worst_case_merk_insert_tree: 0, + worst_case_merk_delete_tree: 0, + worst_case_merk_insert_element: 0, + worst_case_merk_replace_element: 0, + worst_case_merk_patch_element: 0, + worst_case_merk_delete_element: 0, + add_worst_case_has_raw_cost: 0, + add_worst_case_get_raw_tree_cost: 0, + add_worst_case_get_raw_cost: 0, + add_worst_case_get_cost: 0, + }, + }, + path_query_methods: GroveDBPathQueryMethodVersions { + terminal_keys: 0, + merge: 0, + query_items_at_path: 0, + }, + replication: GroveDBReplicationVersions { + get_subtrees_metadata: 0, + fetch_chunk: 0, + start_snapshot_syncing: 0, + apply_chunk: 0, + }, + }, + merk_versions: MerkVersions {}, +}; diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index 9cfe03dba..bd50c796a 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -17,7 +17,7 @@ use nohash_hasher::IntMap; #[cfg(feature = "full")] use crate::{ batch::{key_info::KeyInfo, GroveOp, KeyInfoPath, QualifiedGroveDbOp, TreeCache}, - Element, ElementFlags, Error, + ElementFlags, Error, }; #[cfg(feature = "full")] @@ -124,17 +124,14 @@ where | GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { - if let Element::Tree(..) = element { - cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); - } else if let Element::SumTree(..) = element { - cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, true)); + if let Some(tree_type) = element.tree_type() { + cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, tree_type)); } Ok(()) } GroveOp::RefreshReference { .. } | GroveOp::Delete - | GroveOp::DeleteTree - | GroveOp::DeleteSumTree => Ok(()), + | GroveOp::DeleteTree(_) => Ok(()), GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { Err(Error::InvalidBatchOperation( "replace and insert tree hash are internal operations only", diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 2d25ae81d..0eb4efe06 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -20,7 +20,7 @@ use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; - +use grovedb_merk::merk::TreeType; use crate::Element; #[cfg(feature = "full")] use crate::{ @@ -36,7 +36,7 @@ impl GroveOp { fn worst_case_cost( &self, key: &KeyInfo, - is_in_parent_sum_tree: bool, + in_parent_tree_type: TreeType, worst_case_layer_element_estimates: &WorstCaseLayerInformation, propagate: bool, grove_version: &GroveVersion, @@ -49,20 +49,20 @@ impl GroveOp { } }; match self { - GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::worst_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => GroveDb::worst_case_merk_replace_tree( key, - sum.is_some(), - is_in_parent_sum_tree, + aggregate_data, + in_parent_tree_type, worst_case_layer_element_estimates, propagate, grove_version, ), - GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { + GroveOp::InsertTreeWithRootHash { flags, aggregate_data, .. } => { GroveDb::worst_case_merk_insert_tree( key, flags, - sum.is_some(), - is_in_parent_sum_tree, + aggregate_data, + in_parent_tree_type, propagate_if_input(), grove_version, ) @@ -71,7 +71,7 @@ impl GroveOp { GroveDb::worst_case_merk_insert_element( key, element, - is_in_parent_sum_tree, + in_parent_tree_type, propagate_if_input(), grove_version, ) @@ -88,14 +88,14 @@ impl GroveOp { *max_reference_hop, flags.clone(), ), - is_in_parent_sum_tree, + in_parent_tree_type, propagate_if_input(), grove_version, ), GroveOp::Replace { element } => GroveDb::worst_case_merk_replace_element( key, element, - is_in_parent_sum_tree, + in_parent_tree_type, propagate_if_input(), grove_version, ), @@ -105,7 +105,7 @@ impl GroveOp { } => GroveDb::worst_case_merk_replace_element( key, element, - is_in_parent_sum_tree, + in_parent_tree_type, propagate_if_input(), grove_version, ), @@ -115,16 +115,9 @@ impl GroveOp { propagate, grove_version, ), - GroveOp::DeleteTree => GroveDb::worst_case_merk_delete_tree( - key, - false, - worst_case_layer_element_estimates, - propagate, - grove_version, - ), - GroveOp::DeleteSumTree => GroveDb::worst_case_merk_delete_tree( + GroveOp::DeleteTree(tree_type) => GroveDb::worst_case_merk_delete_tree( key, - true, + tree_type, worst_case_layer_element_estimates, propagate, grove_version, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 106ab910b..87bd59f62 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -66,6 +66,7 @@ use grovedb_version::{ use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; +use grovedb_merk::merk::{NodeType, TreeType}; use grovedb_merk::tree::AggregateData; use key_info::{KeyInfo, KeyInfo::KnownKey}; pub use options::BatchApplyOptions; @@ -75,7 +76,7 @@ pub use crate::batch::batch_structure::{OpsByLevelPath, OpsByPath}; use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, - element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, + element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, TREE_COST_SIZE}, operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, @@ -143,16 +144,14 @@ pub enum GroveOp { /// Delete Delete, /// Delete tree - DeleteTree, - /// Delete sum tree - DeleteSumTree, + DeleteTree(TreeType), } impl GroveOp { fn to_u8(&self) -> u8 { match self { - GroveOp::DeleteTree => 0, - GroveOp::DeleteSumTree => 1, + GroveOp::DeleteTree(_) => 0, + // 1 used to be used for the DeleteSumTree GroveOp::Delete => 2, GroveOp::InsertTreeWithRootHash { .. } => 3, GroveOp::ReplaceTreeRootKey { .. } => 4, @@ -379,8 +378,7 @@ impl fmt::Debug for QualifiedGroveDbOp { ) } GroveOp::Delete => "Delete".to_string(), - GroveOp::DeleteTree => "Delete Tree".to_string(), - GroveOp::DeleteSumTree => "Delete Sum Tree".to_string(), + GroveOp::DeleteTree(tree_type) => format!("Delete Tree {}", tree_type), GroveOp::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(), GroveOp::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(), }; @@ -510,16 +508,12 @@ impl QualifiedGroveDbOp { } /// A delete tree op using a known owned path and known key - pub fn delete_tree_op(path: Vec>, key: Vec, is_sum_tree: bool) -> Self { + pub fn delete_tree_op(path: Vec>, key: Vec, tree_type: TreeType) -> Self { let path = KeyInfoPath::from_known_owned_path(path); Self { path, key: KnownKey(key), - op: if is_sum_tree { - GroveOp::DeleteSumTree - } else { - GroveOp::DeleteTree - }, + op: GroveOp::DeleteTree(tree_type), } } @@ -533,15 +527,11 @@ impl QualifiedGroveDbOp { } /// A delete tree op - pub fn delete_estimated_tree_op(path: KeyInfoPath, key: KeyInfo, is_sum_tree: bool) -> Self { + pub fn delete_estimated_tree_op(path: KeyInfoPath, key: KeyInfo, tree_type: TreeType) -> Self { Self { path, key, - op: if is_sum_tree { - GroveOp::DeleteSumTree - } else { - GroveOp::DeleteTree - }, + op: GroveOp::DeleteTree(tree_type), } } @@ -682,7 +672,7 @@ impl fmt::Debug for TreeCacheMerkByPath { } trait TreeCache { - fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error>; + fn insert(&mut self, op: &QualifiedGroveDbOp, tree_type: TreeType) -> CostResult<(), Error>; fn get_batch_run_mode(&self) -> BatchRunMode; @@ -864,10 +854,10 @@ where /// /// # Returns /// - /// * `Ok((Element, Vec, bool))` - Returns the deserialized `Element` + /// * `Ok((Element, Vec, TreeType))` - Returns the deserialized `Element` /// and the serialized counterpart if the retrieval and deserialization /// are successful, wrapped in the associated cost. Also returns if the - /// merk of the element is a sum tree as a bool. + /// merk of the element is a sum tree as a TreeType. /// * `Err(Error)` - Returns an error if any issue occurs during the /// retrieval or deserialization of the referenced element. /// @@ -884,7 +874,7 @@ where key: &[u8], reference_path: &[Vec], grove_version: &GroveVersion, - ) -> CostResult, bool)>, Error> { + ) -> CostResult, TreeType)>, Error> { let mut cost = OperationCost::default(); let merk = match self.merks.entry(reference_path.to_vec()) { @@ -906,7 +896,7 @@ where .map_err(|e| Error::CorruptedData(e.to_string())) ); - let is_sum_tree = merk.tree_type; + let tree_type = merk.tree_type; if let Some(referenced_element) = referenced_element { let element = cost_return_on_error_no_add!( @@ -916,7 +906,7 @@ where }) ); - Ok(Some((element, referenced_element, is_sum_tree))).wrap_with_cost(cost) + Ok(Some((element, referenced_element, tree_type))).wrap_with_cost(cost) } else { Ok(None).wrap_with_cost(cost) } @@ -1028,7 +1018,7 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1146,7 +1136,7 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) => { + Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) @@ -1175,7 +1165,7 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1201,7 +1191,7 @@ where grove_version, ) } - GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree(_) => { Err(Error::InvalidBatchOperation( "references can not point to something currently being deleted", )) @@ -1233,7 +1223,7 @@ where F: FnMut(&[Vec], bool) -> CostResult, Error>, S: StorageContext<'db>, { - fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, tree_type: TreeType) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let mut inserted_path = op.path.to_path(); @@ -1241,7 +1231,7 @@ where if let HashMapEntry::Vacant(e) = self.merks.entry(inserted_path.clone()) { let mut merk = cost_return_on_error!(&mut cost, (self.get_merk_fn)(&inserted_path, true)); - merk.tree_type = is_sum_tree; + merk.tree_type = tree_type; e.insert(merk); } @@ -1285,7 +1275,7 @@ where let path = &p; // This also populates Merk trees cache - let tree_type = { + let in_tree_type = { let merk = match self.merks.entry(path.to_vec()) { HashMapEntry::Occupied(o) => o.into_mut(), HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( @@ -1307,7 +1297,7 @@ where let merk_feature_type = cost_return_on_error!( &mut cost, element - .get_feature_type(tree_type) + .get_feature_type(in_tree_type) .wrap_with_cost(OperationCost::default()) ); let path_reference = cost_return_on_error!( @@ -1349,11 +1339,11 @@ where ) ); } - Element::Tree(..) | Element::SumTree(..) => { + Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element - .get_feature_type(tree_type) + .get_feature_type(in_tree_type) .wrap_with_cost(OperationCost::default()) ); cost_return_on_error!( @@ -1372,7 +1362,7 @@ where let merk_feature_type = cost_return_on_error!( &mut cost, element - .get_feature_type(tree_type) + .get_feature_type(in_tree_type) .wrap_with_cost(OperationCost::default()) ); if batch_apply_options.validate_insertion_does_not_override { @@ -1451,11 +1441,7 @@ where .wrap_with_cost(cost); }; - let merk_feature_type = if tree_type { - SummedMerkNode(0) - } else { - BasicMerkNode - }; + let merk_feature_type = in_tree_type.empty_tree_feature_type(); let path_reference = cost_return_on_error!( &mut cost, @@ -1502,32 +1488,20 @@ where Element::delete_into_batch_operations( key_info.get_key(), false, - tree_type, /* we are in a sum tree, this might or might not be a + in_tree_type, /* we are in a sum tree, this might or might not be a * sum item */ &mut batch_operations, grove_version ) ); } - GroveOp::DeleteTree => { - cost_return_on_error!( - &mut cost, - Element::delete_into_batch_operations( - key_info.get_key(), - true, - false, - &mut batch_operations, - grove_version - ) - ); - } - GroveOp::DeleteSumTree => { + GroveOp::DeleteTree(tree_type) => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( key_info.get_key(), true, - true, + tree_type, &mut batch_operations, grove_version ) @@ -1556,16 +1530,20 @@ where hash, root_key, flags, - sum, + aggregate_data, } => { - let element = match sum { - None => Element::new_tree_with_flags(root_key, flags), - Some(sum_value) => Element::new_sum_tree_with_flags_and_sum_value( - root_key, sum_value, flags, + let element = match aggregate_data { + AggregateData::NoAggregateData => Element::new_tree_with_flags(root_key, flags), + AggregateData::Sum(sum_value) => Element::new_sum_tree_with_flags_and_sum_value( + root_key, aggregate_data.as_i64(), flags, + ), + AggregateData::BigSum(sum_value) => Element::new_big_sum_tree_with_flags_and_sum_value( + root_key, aggregate_data.as_i128(), flags, ), + AggregateData::Count(count_value) => Element::new_count_tree_with_flags_and_count_value(root_key, aggregate_data.as_u64(), flags) }; let merk_feature_type = - cost_return_on_error_no_add!(&cost, element.get_feature_type(tree_type)); + cost_return_on_error_no_add!(&cost, element.get_feature_type(in_tree_type)); cost_return_on_error!( &mut cost, @@ -1591,7 +1569,7 @@ where &[], Some(batch_apply_options.as_merk_options()), &|key, value| { - Element::specialized_costs_for_key_value(key, value, tree_type, grove_version) + Element::specialized_costs_for_key_value(key, value, in_tree_type.inner_node_type(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), @@ -1643,11 +1621,13 @@ where // we need to give back the value defined cost in the case that the // new element is a tree match new_element { - Element::Tree(..) | Element::SumTree(..) => { - let tree_cost_size = if new_element.is_sum_tree() { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE + Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + let tree_type = new_element.tree_type().unwrap(); + let tree_cost_size = match tree_type { + TreeType::NormalTree => TREE_COST_SIZE, + TreeType::SumTree => SUM_TREE_COST_SIZE, + TreeType::BigSumTree => BIG_SUM_TREE_COST_SIZE, + TreeType::CountTree => COUNT_TREE_COST_SIZE, }; let tree_value_cost = tree_cost_size + flags_len @@ -1822,7 +1802,7 @@ impl GroveDb { } => { *hash = root_hash; *root_key = calculated_root_key; - *aggregate_data = aggregate_data; + *aggregate_data = *aggregate_data; } GroveOp::InsertTreeWithRootHash { .. } => { return Err(Error::CorruptedCodeExecution( @@ -1869,8 +1849,7 @@ impl GroveDb { .wrap_with_cost(cost); } GroveOp::Delete - | GroveOp::DeleteTree - | GroveOp::DeleteSumTree => { + | GroveOp::DeleteTree(_) => { if calculated_root_key.is_some() { return Err(Error::InvalidBatchOperation( "modification of tree when it will be \ @@ -2161,7 +2140,7 @@ impl GroveDb { if let Some((parent_path, parent_key)) = path.derive_parent() { if new_merk { // TODO: can this be a sum tree - Ok(Merk::open_empty(storage, MerkType::LayeredMerk, false)).wrap_with_cost(cost) + Ok(Merk::open_empty(storage, MerkType::LayeredMerk, TreeType::NormalTree)).wrap_with_cost(cost) } else { let parent_storage = self .db @@ -2184,12 +2163,11 @@ impl GroveDb { } ) ); - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { + if let Some((root_key, tree_type)) = element.root_key_and_tree_type_owned() { Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -2205,11 +2183,11 @@ impl GroveDb { } } } else if new_merk { - Ok(Merk::open_empty(storage, MerkType::BaseMerk, false)).wrap_with_cost(cost) + Ok(Merk::open_empty(storage, MerkType::BaseMerk, TreeType::NormalTree)).wrap_with_cost(cost) } else { Merk::open_base( storage, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -2245,7 +2223,7 @@ impl GroveDb { } else { MerkType::LayeredMerk }; - Ok(Merk::open_empty(storage, merk_type, false)).wrap_with_cost(local_cost) + Ok(Merk::open_empty(storage, merk_type, TreeType::NormalTree)).wrap_with_cost(local_cost) } else if let Some((base_path, last)) = path.derive_parent() { let parent_storage = self .db @@ -2255,12 +2233,11 @@ impl GroveDb { &mut local_cost, Element::get_from_storage(&parent_storage, last, grove_version) ); - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { + if let Some((root_key, tree_type)) = element.root_key_and_tree_type_owned() { Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -2277,7 +2254,7 @@ impl GroveDb { } else { Merk::open_base( storage, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index e2b3d9a9c..fceccce6e 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -9,7 +9,7 @@ mod tests { }; use grovedb_version::version::GroveVersion; use intmap::IntMap; - + use grovedb_merk::merk::TreeType; use crate::{ batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, @@ -75,7 +75,7 @@ mod tests { let ops = vec![QualifiedGroveDbOp::delete_tree_op( vec![], b"key1".to_vec(), - false, + TreeType::NormalTree, )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) @@ -219,7 +219,7 @@ mod tests { let ops = vec![QualifiedGroveDbOp::delete_tree_op( vec![], b"key1".to_vec(), - false, + TreeType::NormalTree, )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) @@ -368,7 +368,7 @@ mod tests { let ops = vec![QualifiedGroveDbOp::delete_tree_op( vec![], b"key1".to_vec(), - false, + TreeType::NormalTree, )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) @@ -467,7 +467,7 @@ mod tests { let ops = vec![QualifiedGroveDbOp::delete_tree_op( vec![], b"key1".to_vec(), - false, + TreeType::NormalTree, )]; let batch_cost = db .apply_batch_with_element_flags_update( @@ -642,7 +642,7 @@ mod tests { let ops = vec![QualifiedGroveDbOp::delete_tree_op( vec![], b"key1".to_vec(), - false, + TreeType::NormalTree, )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) diff --git a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs index c0be9a082..69391631a 100644 --- a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs @@ -2,6 +2,7 @@ #[cfg(feature = "full")] mod tests { + use grovedb_merk::merk::TreeType; use grovedb_version::version::GroveVersion; use crate::{ @@ -46,7 +47,7 @@ mod tests { let ops = vec![QualifiedGroveDbOp::delete_tree_op( vec![], b"key1".to_vec(), - false, + TreeType::NormalTree, )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) @@ -153,7 +154,7 @@ mod tests { let ops = vec![QualifiedGroveDbOp::delete_tree_op( vec![], b"key1".to_vec(), - false, + TreeType::NormalTree, )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 75e051f54..3cc381a62 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -7,7 +7,7 @@ use crate::{ reference_path::ReferencePathType, Element, ElementFlags, }; -use crate::element::BigSumValue; +use crate::element::{BigSumValue, CountValue}; impl Element { #[cfg(feature = "full")] @@ -140,7 +140,7 @@ impl Element { } #[cfg(feature = "full")] - /// Set element to a sum tree with flags + /// Set element to a big sum tree with flags pub fn new_big_sum_tree_with_flags( maybe_root_key: Option>, flags: Option, @@ -149,7 +149,7 @@ impl Element { } #[cfg(feature = "full")] - /// Set element to a sum tree with flags and sum value + /// Set element to a big sum tree with flags and sum value pub fn new_big_sum_tree_with_flags_and_sum_value( maybe_root_key: Option>, big_sum_value: BigSumValue, @@ -157,4 +157,29 @@ impl Element { ) -> Self { Element::BigSumTree(maybe_root_key, big_sum_value, flags) } + + #[cfg(feature = "full")] + /// Set element to a count tree without flags + pub fn new_count_tree(maybe_root_key: Option>) -> Self { + Element::CountTree(maybe_root_key, 0, None) + } + + #[cfg(feature = "full")] + /// Set element to a count tree with flags + pub fn new_count_tree_with_flags( + maybe_root_key: Option>, + flags: Option, + ) -> Self { + Element::CountTree(maybe_root_key, 0, flags) + } + + #[cfg(feature = "full")] + /// Set element to a count tree with flags and sum value + pub fn new_count_tree_with_flags_and_count_value( + maybe_root_key: Option>, + count_value: CountValue, + flags: Option, + ) -> Self { + Element::CountTree(maybe_root_key, count_value, flags) + } } diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index 9382c435a..af2992f1f 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -7,6 +7,7 @@ use grovedb_costs::OperationCost; use grovedb_costs::{storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt}; #[cfg(feature = "full")] use grovedb_merk::{BatchEntry, Error as MerkError, Merk, MerkOptions, Op}; +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_storage::StorageContext; #[cfg(feature = "full")] @@ -27,24 +28,28 @@ impl Element { key: K, merk_options: Option, is_layered: bool, - is_sum: bool, + in_tree_type: TreeType, grove_version: &GroveVersion, ) -> CostResult<(), Error> { check_grovedb_v0_with_cost!("delete", grove_version.grovedb_versions.element.delete); - let op = match (is_sum, is_layered) { - (true, true) => Op::DeleteLayeredMaybeSpecialized, - (true, false) => Op::DeleteMaybeSpecialized, - (false, true) => Op::DeleteLayered, - (false, false) => Op::Delete, + let op = match (in_tree_type, is_layered) { + (TreeType::NormalTree, true) => Op::DeleteLayered, + (TreeType::NormalTree, false) => Op::Delete, + (TreeType::SumTree, true) + | (TreeType::BigSumTree, true) + | (TreeType::CountTree, true) => Op::DeleteLayeredMaybeSpecialized, + (TreeType::SumTree, false) + | (TreeType::BigSumTree, false) + | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; - let uses_sum_nodes = merk.tree_type; + let tree_type = merk.tree_type; //todo not sure we get it again, we need to see if this is necessary merk.apply_with_specialized_costs::<_, Vec>( &batch, &[], merk_options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, uses_sum_nodes, grove_version) + Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), @@ -60,7 +65,7 @@ impl Element { key: K, merk_options: Option, is_layered: bool, - is_in_sum_tree: bool, + in_tree_type: TreeType, sectioned_removal: &mut impl FnMut( &Vec, u32, @@ -78,20 +83,24 @@ impl Element { .element .delete_with_sectioned_removal_bytes ); - let op = match (is_in_sum_tree, is_layered) { - (true, true) => Op::DeleteLayeredMaybeSpecialized, - (true, false) => Op::DeleteMaybeSpecialized, - (false, true) => Op::DeleteLayered, - (false, false) => Op::Delete, + let op = match (in_tree_type, is_layered) { + (TreeType::NormalTree, true) => Op::DeleteLayered, + (TreeType::NormalTree, false) => Op::Delete, + (TreeType::SumTree, true) + | (TreeType::BigSumTree, true) + | (TreeType::CountTree, true) => Op::DeleteLayeredMaybeSpecialized, + (TreeType::SumTree, false) + | (TreeType::BigSumTree, false) + | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; - let uses_sum_nodes = merk.tree_type; + let tree_type = merk.tree_type; //todo not sure we get it again, we need to see if this is necessary merk.apply_with_costs_just_in_time_value_update::<_, Vec>( &batch, &[], merk_options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, uses_sum_nodes, grove_version) + Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), @@ -108,7 +117,7 @@ impl Element { pub fn delete_into_batch_operations>( key: K, is_layered: bool, - is_sum: bool, + in_tree_type: TreeType, batch_operations: &mut Vec>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -119,11 +128,15 @@ impl Element { .element .delete_into_batch_operations ); - let op = match (is_sum, is_layered) { - (true, true) => Op::DeleteLayeredMaybeSpecialized, - (true, false) => Op::DeleteMaybeSpecialized, - (false, true) => Op::DeleteLayered, - (false, false) => Op::Delete, + let op = match (in_tree_type, is_layered) { + (TreeType::NormalTree, true) => Op::DeleteLayered, + (TreeType::NormalTree, false) => Op::Delete, + (TreeType::SumTree, true) + | (TreeType::BigSumTree, true) + | (TreeType::CountTree, true) => Op::DeleteLayeredMaybeSpecialized, + (TreeType::SumTree, false) + | (TreeType::BigSumTree, false) + | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, }; let entry = (key, op); batch_operations.push(entry); diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index ded859d36..876971a79 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -16,8 +16,8 @@ use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; - -use crate::element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}; +use grovedb_merk::merk::NodeType; +use crate::element::{CostSize, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}; #[cfg(feature = "full")] use crate::{Element, Error, Hash}; @@ -118,13 +118,28 @@ impl Element { key: K, grove_version: &GroveVersion, ) -> CostResult, Error> { - check_grovedb_v0_with_cost!( - "get_optional_from_storage", - grove_version - .grovedb_versions - .element - .get_optional_from_storage - ); + match grove_version + .grovedb_versions + .element + .get_optional_from_storage { + 0 => Self::get_optional_from_storage_v0(storage, key, grove_version), + 1 => Self::get_optional_from_storage_v1(storage, key, grove_version), + version => Err(Error::VersionError(GroveVersionError::UnknownVersionMismatch { + method: "get_optional_from_storage".to_string(), + known_versions: vec![0, 1], + received: version, + })).wrap_with_cost(OperationCost::default()) + } + } + + #[cfg(feature = "full")] + /// Get an element directly from storage under a key + /// Merk does not need to be loaded + fn get_optional_from_storage_v0<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( + storage: &S, + key: K, + grove_version: &GroveVersion, + ) -> CostResult, Error> { let mut cost = OperationCost::default(); let key_ref = key.as_ref(); let node_value_opt = cost_return_on_error!( @@ -162,7 +177,7 @@ impl Element { cost.storage_loaded_bytes = KV::value_byte_cost_size_for_key_and_value_lengths( key_ref.len() as u32, value.as_ref().unwrap().len() as u32, - false, + NodeType::NormalNode, ) as u64 } Some(Element::SumItem(_, flags)) => { @@ -173,14 +188,10 @@ impl Element { }); let value_len = cost_size + flags_len; cost.storage_loaded_bytes = - KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false) as u64 + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, NodeType::NormalNode) as u64 } - Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags)) => { - let tree_cost_size = if element.as_ref().unwrap().is_sum_tree() { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags )) | Some(Element::BigSumTree(_, _, flags )) | Some(Element::CountTree(_, _, flags )) => { + let tree_cost_size = element.as_ref().unwrap().tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 @@ -190,7 +201,7 @@ impl Element { KV::layered_value_byte_cost_size_for_key_and_value_lengths( key_ref.len() as u32, value_len, - false, + NodeType::NormalNode, ) as u64 } None => {} @@ -198,6 +209,81 @@ impl Element { Ok(element).wrap_with_cost(cost) } + + #[cfg(feature = "full")] + /// Get an element directly from storage under a key + /// Merk does not need to be loaded + fn get_optional_from_storage_v1<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( + storage: &S, + key: K, + grove_version: &GroveVersion, + ) -> CostResult, Error> { + let mut cost = OperationCost::default(); + let key_ref = key.as_ref(); + let node_value_opt = cost_return_on_error!( + &mut cost, + storage + .get(key_ref) + .map_err(|e| Error::CorruptedData(e.to_string())) + ); + let maybe_tree_inner: Option = cost_return_on_error_no_add!( + &cost, + node_value_opt + .map(|node_value| { + Decode::decode(node_value.as_slice()) + .map_err(|e| Error::CorruptedData(e.to_string())) + }) + .transpose() + ); + + let Some((value, tree_feature_type)) = maybe_tree_inner.map(|tree_inner| tree_inner.value_as_owned_with_feature()) else { + return Ok(None).wrap_with_cost(cost) + }; + let node_type = tree_feature_type.node_type(); + let element = cost_return_on_error_no_add!( + &cost, + Self::deserialize(value.as_slice(), grove_version).map_err(|_| { + Error::CorruptedData(String::from("unable to deserialize element")) + }) + ); + match &element { + Element::Item(..) | Element::Reference(..) => { + // while the loaded item might be a sum item, it is given for free + // as it would be very hard to know in advance + cost.storage_loaded_bytes = KV::value_byte_cost_size_for_key_and_value_lengths( + key_ref.len() as u32, + value.len() as u32, + node_type, + ) as u64 + } + Element::SumItem(_, flags) => { + let cost_size = SUM_ITEM_COST_SIZE; + let flags_len = flags.as_ref().map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let value_len = cost_size + flags_len; + cost.storage_loaded_bytes = + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, node_type) as u64 // this is changed to sum node in v1 + } + Element::Tree(_, flags) | Element::SumTree(_, _, flags ) | Element::BigSumTree(_, _, flags ) | Element::CountTree(_, _, flags ) => { + let tree_cost_size = element.tree_type().unwrap().cost_size(); + let flags_len = flags.as_ref().map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let value_len = tree_cost_size + flags_len; + cost.storage_loaded_bytes = + KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key_ref.len() as u32, + value_len, + node_type, + ) as u64 + } + } + Ok(Some(element)).wrap_with_cost(cost) + } + #[cfg(feature = "full")] /// Get an element from Merk under a key; path should be resolved and proper /// Merk should be loaded by this moment @@ -262,6 +348,7 @@ impl Element { #[cfg(feature = "full")] #[cfg(test)] mod tests { + use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage, StorageBatch}; @@ -277,7 +364,7 @@ mod tests { .unwrap(); let mut merk = Merk::open_base( ctx, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -302,7 +389,7 @@ mod tests { .unwrap(); let mut merk = Merk::open_base( ctx, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 20022d6fd..79c9205c2 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -16,7 +16,7 @@ use grovedb_merk::{ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::{NodeType, TreeType}; use grovedb_merk::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; @@ -122,6 +122,30 @@ impl Element { matches!(self, Element::SumTree(..)) } + #[cfg(any(feature = "full", feature = "verify"))] + /// Check if the element is a tree and return the root_tree info and tree type + pub fn root_key_and_tree_type_owned(self) -> Option<(Option>, TreeType)> { + match self { + Element::Tree(root_key, _) => Some((root_key, TreeType::NormalTree)), + Element::SumTree(root_key, _, _) => Some((root_key, TreeType::SumTree)), + Element::BigSumTree(root_key, _, _) => Some((root_key, TreeType::BigSumTree)), + Element::CountTree(root_key, _, _) => Some((root_key, TreeType::CountTree)), + _ => None, + } + } + + #[cfg(any(feature = "full", feature = "verify"))] + /// Check if the element is a tree and return the root_tree info and the tree type + pub fn root_key_and_tree_type(&self) -> Option<(&Option>, TreeType)> { + match self { + Element::Tree(root_key, _) => Some((root_key, TreeType::NormalTree)), + Element::SumTree(root_key, _, _) => Some((root_key, TreeType::SumTree)), + Element::BigSumTree(root_key, _, _) => Some((root_key, TreeType::BigSumTree)), + Element::CountTree(root_key, _, _) => Some((root_key, TreeType::CountTree)), + _ => None, + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a tree and return the tree type pub fn tree_type(&self) -> Option { @@ -129,6 +153,7 @@ impl Element { Element::Tree(_, _) => Some(TreeType::NormalTree), Element::SumTree(_, _, _) => Some(TreeType::SumTree), Element::BigSumTree(_, _, _) => Some(TreeType::BigSumTree), + Element::CountTree(_, _, _) => Some(TreeType::CountTree), _ => None, } } @@ -148,7 +173,7 @@ impl Element { #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a tree pub fn is_any_tree(&self) -> bool { - matches!(self, Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..)) + matches!(self, Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..)) } #[cfg(any(feature = "full", feature = "verify"))] @@ -194,6 +219,8 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) + | Element::BigSumTree(.., flags) + | Element::CountTree(.., flags) | Element::SumItem(_, flags) => flags, } } @@ -206,6 +233,8 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) + | Element::BigSumTree(.., flags) + | Element::CountTree(.., flags) | Element::SumItem(_, flags) => flags, } } @@ -218,6 +247,8 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) + | Element::BigSumTree(.., flags) + | Element::CountTree(.., flags) | Element::SumItem(_, flags) => flags, } } @@ -230,6 +261,8 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) + | Element::BigSumTree(.., flags) + | Element::CountTree(.., flags) | Element::SumItem(_, flags) => *flags = new_flags, } } @@ -284,7 +317,7 @@ impl Element { pub fn specialized_costs_for_key_value( key: &Vec, value: &[u8], - is_sum_node: bool, + node_type: NodeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -307,7 +340,7 @@ impl Element { KV::layered_value_byte_cost_size_for_key_and_value_lengths( key_len, value_len, - is_sum_node, + node_type, ) } Element::SumTree(_, _sum_value, flags) => { @@ -320,7 +353,7 @@ impl Element { KV::layered_value_byte_cost_size_for_key_and_value_lengths( key_len, value_len, - is_sum_node, + node_type, ) } Element::BigSumTree(_, _sum_value, flags) => { @@ -333,7 +366,7 @@ impl Element { KV::layered_value_byte_cost_size_for_key_and_value_lengths( key_len, value_len, - is_sum_node, + node_type, ) } Element::SumItem(.., flags) => { @@ -343,9 +376,9 @@ impl Element { }); let value_len = SUM_ITEM_COST_SIZE + flags_len; let key_len = key.len() as u32; - KV::node_value_byte_cost_size(key_len, value_len, is_sum_node) + KV::node_value_byte_cost_size(key_len, value_len, node_type) } - _ => KV::node_value_byte_cost_size(key.len() as u32, value.len() as u32, is_sum_node), + _ => KV::node_value_byte_cost_size(key.len() as u32, value.len() as u32, node_type), }; Ok(cost) } diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 97a218a89..2dd56da5f 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -32,7 +32,7 @@ impl Element { let serialized = cost_return_on_error_default!(self.serialize(grove_version)); - if !merk.tree_type && self.is_sum_item() { + if !merk.tree_type.allows_sum_item() && self.is_sum_item() { return Err(Error::InvalidInput("cannot add sum item to non sum tree")) .wrap_with_cost(Default::default()); } @@ -55,14 +55,14 @@ impl Element { } else { [(key, Op::Put(serialized, merk_feature_type))] }; - let uses_sum_nodes = merk.tree_type; + let tree_type = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch_operations, &[], options, &|key, value| { // it is possible that a normal item was being replaced with a - Self::specialized_costs_for_key_value(key, value, uses_sum_nodes, grove_version) + Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), @@ -314,13 +314,13 @@ impl Element { key, Op::PutCombinedReference(serialized, referenced_value, merk_feature_type), )]; - let uses_sum_nodes = merk.tree_type; + let tree_type = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch_operations, &[], options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, uses_sum_nodes, grove_version) + Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), @@ -401,13 +401,13 @@ impl Element { key, Op::PutLayeredReference(serialized, cost, subtree_root_hash, merk_feature_type), )]; - let uses_sum_nodes = merk.tree_type; + let tree_type = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch_operations, &[], options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, uses_sum_nodes, grove_version) + Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 6abdaa19c..b09844588 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -27,8 +27,9 @@ use bincode::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; #[cfg(feature = "full")] -use grovedb_merk::estimated_costs::{LAYER_COST_SIZE, SUM_LAYER_COST_SIZE}; -use grovedb_merk::estimated_costs::{BIG_SUM_LAYER_COST_SIZE, BIG_SUM_VALUE_EXTRA_COST}; +use grovedb_merk::estimated_costs::{LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, BIG_SUM_LAYER_COST_SIZE, BIG_SUM_VALUE_EXTRA_COST}; +#[cfg(feature = "full")] +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_visualize::visualize_to_vec; @@ -65,6 +66,10 @@ pub const SUM_TREE_COST_SIZE: u32 = SUM_LAYER_COST_SIZE; // 12 /// The cost of a big sum tree pub const BIG_SUM_TREE_COST_SIZE: u32 = BIG_SUM_LAYER_COST_SIZE; // 19 +#[cfg(feature = "full")] +/// The cost of a count tree +pub const COUNT_TREE_COST_SIZE: u32 = SUM_LAYER_COST_SIZE; // 12 + #[cfg(any(feature = "full", feature = "verify"))] /// int 64 sum value pub type SumValue = i64; @@ -77,6 +82,22 @@ pub type BigSumValue = i128; /// int 64 count value pub type CountValue = u64; +pub trait CostSize { + fn cost_size(&self) -> u32; +} + +impl CostSize for TreeType { + fn cost_size(&self) -> u32 { + match self { + TreeType::NormalTree => TREE_COST_SIZE, + TreeType::SumTree => SUM_TREE_COST_SIZE, + TreeType::BigSumTree => BIG_SUM_TREE_COST_SIZE, + TreeType::CountTree => COUNT_TREE_COST_SIZE, + } + } +} + + #[cfg(any(feature = "full", feature = "verify"))] /// Variants of GroveDB stored entities /// @@ -102,9 +123,9 @@ pub enum Element { /// nodes in big form i128 /// The big sum tree is valuable if you have a big sum tree of sum trees BigSumTree(Option>, BigSumValue, Option), - // /// Same as Element::Tree but underlying Merk counts value of its countable - // /// nodes - // CountTree(Option>, CountValue, Option), + /// Same as Element::Tree but underlying Merk counts value of its countable + /// nodes + CountTree(Option>, CountValue, Option), } impl fmt::Display for Element { @@ -173,6 +194,17 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::CountTree(root_key, count_value, flags) => { + write!( + f, + "CountTree({}, {}{})", + root_key.as_ref().map_or("None".to_string(), hex::encode), + count_value, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } } } } @@ -186,6 +218,7 @@ impl Element { Element::SumItem(..) => "sum item", Element::SumTree(..) => "sum tree", Element::BigSumTree(..) => "big sum tree", + Element::CountTree(..) => "count tree", } } diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 7226db553..5a94797c4 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -22,6 +22,8 @@ use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, StorageConte use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; +#[cfg(feature = "full")] +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index 4c409b999..d745669dc 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -22,12 +22,13 @@ use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; - +use grovedb_merk::merk::TreeType; use crate::{ batch::{key_info::KeyInfo, KeyInfoPath}, element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, Element, ElementFlags, Error, GroveDb, }; +use crate::element::CostSize; pub const WORST_CASE_FLAGS_LEN: u32 = 16386; // 2 bytes to represent this number for varint @@ -36,7 +37,7 @@ impl GroveDb { pub fn add_worst_case_get_merk_at_path<'db, S: Storage<'db>>( cost: &mut OperationCost, path: &KeyInfoPath, - is_sum_tree: bool, + tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -55,7 +56,7 @@ impl GroveDb { cost.storage_loaded_bytes += TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, HASH_LENGTH as u32, - is_sum_tree, + tree_type.inner_node_type(), //todo This is probably wrong ) as u64; } } @@ -66,8 +67,8 @@ impl GroveDb { /// Add worst case for insertion into merk pub(crate) fn worst_case_merk_replace_tree( key: &KeyInfo, - is_sum_tree: bool, - is_in_parent_sum_tree: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, worst_case_layer_information: &WorstCaseLayerInformation, propagate: bool, grove_version: &GroveVersion, @@ -83,17 +84,13 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; - let tree_cost = if is_sum_tree { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost = tree_type.cost_size(); let layer_extra_size = tree_cost + WORST_CASE_FLAGS_LEN; add_worst_case_merk_replace_layered( &mut cost, key_len, layer_extra_size, - is_in_parent_sum_tree, + in_parent_tree_type.inner_node_type(), ); if propagate { add_worst_case_merk_propagate(&mut cost, worst_case_layer_information) @@ -108,8 +105,8 @@ impl GroveDb { pub fn worst_case_merk_insert_tree( key: &KeyInfo, flags: &Option, - is_sum_tree: bool, - is_in_parent_sum_tree: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, propagate_if_input: Option<&WorstCaseLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -128,13 +125,9 @@ impl GroveDb { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); - let tree_cost = if is_sum_tree { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost = tree_type.cost_size(); let value_len = tree_cost + flags_len; - add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, is_in_parent_sum_tree); + add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_parent_tree_type); if let Some(input) = propagate_if_input { add_worst_case_merk_propagate(&mut cost, input).map_err(Error::MerkError) } else { @@ -146,7 +139,7 @@ impl GroveDb { /// Add worst case for insertion into merk pub fn worst_case_merk_delete_tree( key: &KeyInfo, - is_sum_tree: bool, + tree_type: TreeType, worst_case_layer_information: &WorstCaseLayerInformation, propagate: bool, grove_version: &GroveVersion, @@ -162,11 +155,7 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; - let tree_cost = if is_sum_tree { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost = tree_type.cost_size(); let layer_extra_size = tree_cost + WORST_CASE_FLAGS_LEN; add_worst_case_merk_delete_layered(&mut cost, key_len, layer_extra_size); if propagate { @@ -184,7 +173,7 @@ impl GroveDb { pub fn worst_case_merk_insert_element( key: &KeyInfo, value: &Element, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, propagate_for_level: Option<&WorstCaseLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -200,29 +189,25 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; match value { - Element::Tree(_, flags) | Element::SumTree(_, _, flags) => { + Element::Tree(_, flags) | Element::SumTree(_, _, flags) | Element::BigSumTree(_, _, flags) | Element::CountTree(_, _, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); - let tree_cost_size = if value.is_sum_tree() { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost_size = value.tree_type().unwrap().cost_size(); let value_len = tree_cost_size + flags_len; add_cost_case_merk_insert_layered( &mut cost, key_len, value_len, - in_parent_tree_using_sums, + in_parent_tree_type, ) } _ => add_cost_case_merk_insert( &mut cost, key_len, cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, - in_parent_tree_using_sums, + in_parent_tree_type, ), }; if let Some(level) = propagate_for_level { @@ -239,7 +224,7 @@ impl GroveDb { pub fn worst_case_merk_replace_element( key: &KeyInfo, value: &Element, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, propagate_for_level: Option<&WorstCaseLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -270,7 +255,7 @@ impl GroveDb { &mut cost, key_len, value_len, - in_parent_tree_using_sums, + in_parent_tree_type, ) } Element::SumItem(_, flags) => { @@ -283,14 +268,14 @@ impl GroveDb { &mut cost, key_len, value_len, - in_parent_tree_using_sums, + in_parent_tree_type, ) } _ => add_cost_case_merk_replace( &mut cost, key_len, cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, - in_parent_tree_using_sums, + in_parent_tree_type, ), }; if let Some(level) = propagate_for_level { @@ -308,7 +293,7 @@ impl GroveDb { key: &KeyInfo, value: &Element, change_in_bytes: i32, - in_tree_using_sums: bool, + in_parent_tree_type: TreeType, propagate_for_level: Option<&WorstCaseLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -339,7 +324,7 @@ impl GroveDb { key_len, value_len, change_in_bytes, - in_tree_using_sums, + in_parent_tree_type, ) } _ => { @@ -389,7 +374,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, max_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -404,7 +389,7 @@ impl GroveDb { let value_size = TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ); cost.seek_count += 1; cost.storage_loaded_bytes += value_size as u64; @@ -417,8 +402,8 @@ impl GroveDb { cost: &mut OperationCost, _path: &KeyInfoPath, key: &KeyInfo, - is_sum_tree: bool, - in_parent_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -431,16 +416,12 @@ impl GroveDb { ); cost.seek_count += 1; - let tree_cost_size = if is_sum_tree { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost_size = tree_type.cost_size(); add_worst_case_get_merk_node( cost, key.max_length() as u32, tree_cost_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ) .map_err(Error::MerkError) } @@ -451,7 +432,7 @@ impl GroveDb { _path: &KeyInfoPath, key: &KeyInfo, max_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -468,7 +449,7 @@ impl GroveDb { cost, key.max_length() as u32, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ) .map_err(Error::MerkError) } @@ -479,7 +460,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, max_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, max_references_sizes: Vec, grove_version: &GroveVersion, ) -> Result<(), Error> { @@ -496,7 +477,7 @@ impl GroveDb { let value_size: u32 = TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ); cost.seek_count += 1 + max_references_sizes.len() as u32; cost.storage_loaded_bytes += @@ -523,7 +504,7 @@ mod test { }; use grovedb_version::version::GroveVersion; use tempfile::TempDir; - + use grovedb_merk::merk::{NodeType, TreeType}; use crate::{ batch::{key_info::KeyInfo::KnownKey, KeyInfoPath}, tests::{common::EMPTY_PATH, TEST_LEAF}, @@ -569,7 +550,7 @@ mod test { // (this will be the max_element_size) let mut cost = OperationCost::default(); let key = KnownKey(8_u64.to_be_bytes().to_vec()); - add_worst_case_get_merk_node(&mut cost, key.max_length() as u32, 60, false) + add_worst_case_get_merk_node(&mut cost, key.max_length() as u32, 60, NodeType::NormalNode) .expect("no issue with version"); assert_eq!(cost, node_result.cost); } @@ -635,7 +616,7 @@ mod test { &path, &key, elem.serialized_size(grove_version).expect("expected size") as u32, - false, + TreeType::NormalTree, GroveVersion::latest(), ) .expect("expected to add cost"); diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 132f0dbb6..f89bd9c2a 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -319,12 +319,11 @@ impl GroveDb { } ) ); - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { + if let Some((root_key, tree_type)) = element.root_key_and_tree_type_owned() { Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -341,7 +340,7 @@ impl GroveDb { } else { Merk::open_base( storage, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -381,7 +380,7 @@ impl GroveDb { } else { Merk::open_base( storage, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -1130,7 +1129,7 @@ impl GroveDb { while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; match element { - Element::SumTree(..) | Element::Tree(..) => { + Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -1274,7 +1273,7 @@ impl GroveDb { while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; match element { - Element::SumTree(..) | Element::Tree(..) => { + Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, diff --git a/grovedb/src/operations/delete/delete_up_tree.rs b/grovedb/src/operations/delete/delete_up_tree.rs index 7ecfce83e..e2ca1dbe2 100644 --- a/grovedb/src/operations/delete/delete_up_tree.rs +++ b/grovedb/src/operations/delete/delete_up_tree.rs @@ -5,6 +5,7 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; +use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, @@ -169,7 +170,7 @@ impl GroveDb { path: SubtreePath, key: &[u8], options: &DeleteUpTreeOptions, - is_known_to_be_subtree_with_sum: Option<(bool, bool)>, + is_known_to_be_subtree: Option, mut current_batch_operations: Vec, transaction: TransactionArg, grove_version: &GroveVersion, @@ -186,7 +187,7 @@ impl GroveDb { path, key, options, - is_known_to_be_subtree_with_sum, + is_known_to_be_subtree, &mut current_batch_operations, transaction, grove_version, @@ -201,7 +202,7 @@ impl GroveDb { path: SubtreePath, key: &[u8], options: &DeleteUpTreeOptions, - is_known_to_be_subtree_with_sum: Option<(bool, bool)>, + is_known_to_be_subtree: Option, current_batch_operations: &mut Vec, transaction: TransactionArg, grove_version: &GroveVersion, @@ -234,7 +235,7 @@ impl GroveDb { path.clone(), key, &options.to_delete_options(), - is_known_to_be_subtree_with_sum, + is_known_to_be_subtree, current_batch_operations, transaction, grove_version, diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 461d3cf2c..ec8ef98fb 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -21,6 +21,7 @@ use grovedb_costs::{ use grovedb_merk::{proofs::Query, KVIterator}; #[cfg(feature = "full")] use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; +use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_storage::{ @@ -511,7 +512,7 @@ impl GroveDb { path: SubtreePath, key: &[u8], options: &DeleteOptions, - is_known_to_be_subtree_with_sum: Option<(bool, bool)>, + is_known_to_be_subtree: Option, current_batch_operations: &[QualifiedGroveDbOp], transaction: TransactionArg, grove_version: &GroveVersion, @@ -544,28 +545,24 @@ impl GroveDb { ) ); } - let (is_subtree, is_subtree_with_sum) = match is_known_to_be_subtree_with_sum { + let tree_type = match is_known_to_be_subtree { None => { let element = cost_return_on_error!( &mut cost, self.get_raw(path.clone(), key.as_ref(), transaction, grove_version) ); - match element { - Element::Tree(..) => (true, false), - Element::SumTree(..) => (true, true), - _ => (false, false), - } + element.tree_type() } - Some(x) => x, + Some(x) => Some(x), }; - if is_subtree { + if let Some(tree_type) = tree_type { let subtree_merk_path = path.derive_owned_with_child(key); let subtree_merk_path_vec = subtree_merk_path.to_vec(); let batch_deleted_keys = current_batch_operations .iter() .filter_map(|op| match op.op { - GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree(_) => { // todo: to_path clones (best to figure out how to compare without // cloning) if op.path.to_path() == subtree_merk_path_vec { @@ -595,7 +592,7 @@ impl GroveDb { // If there is any current batch operation that is inserting something in this // tree then it is not empty either is_empty &= !current_batch_operations.iter().any(|op| match op.op { - GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => false, + GroveOp::Delete | GroveOp::DeleteTree(_) => false, // todo: fix for to_path (it clones) _ => op.path.to_path() == subtree_merk_path_vec, }); @@ -613,7 +610,7 @@ impl GroveDb { Ok(Some(QualifiedGroveDbOp::delete_tree_op( path.to_vec(), key.to_vec(), - is_subtree_with_sum, + tree_type, ))) } else { Err(Error::NotSupported( @@ -712,7 +709,7 @@ impl GroveDb { ) ); let uses_sum_tree = subtree_to_delete_from.tree_type; - if element.is_any_tree() { + if let Some(tree_type) = element.tree_type() { let subtree_merk_path = path.derive_owned_with_child(key); let subtree_merk_path_ref = SubtreePath::from(&subtree_merk_path); @@ -771,7 +768,7 @@ impl GroveDb { Merk::open_layered_with_root_key( storage, subtree_to_delete_from.root_key(), - element.is_sum_tree(), + tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 07c349991..9548286e4 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -6,7 +6,7 @@ use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_version::error::GroveVersionError; use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; - +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use crate::{util::merk_optional_tx, Element, Error, GroveDb, TransactionArg}; diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 1ac09c8bd..7ff21910a 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -308,7 +308,7 @@ impl GroveDb { println!("lower layer had key {}", hex_to_ascii(key)); } match element { - Element::Tree(Some(_), _) | Element::SumTree(Some(_), ..) => { + Element::Tree(Some(_), _) | Element::SumTree(Some(_), ..) | Element::BigSumTree(Some(_), ..) | Element::CountTree(Some(_), ..) => { path.push(key); let lower_hash = Self::verify_layer_proof( lower_layer, @@ -337,6 +337,8 @@ impl GroveDb { } Element::Tree(None, _) | Element::SumTree(None, ..) + | Element::BigSumTree(None, ..) + | Element::CountTree(None, ..) | Element::SumItem(..) | Element::Item(..) | Element::Reference(..) => { diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 1cd505192..9bb559c56 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -3,6 +3,7 @@ mod state_sync_session; use std::pin::Pin; use grovedb_merk::{tree::hash::CryptoHash, ChunkProducer}; +use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -16,7 +17,7 @@ use crate::{Error, GroveDb, TransactionArg}; /// - `Option>`: The root key, which may be `None` if not present. /// - `bool`: Indicates whether the tree is a sum tree. /// - `Vec`: The chunk ID representing traversal instructions. -pub type ChunkIdentifier = (crate::SubtreePrefix, Option>, bool, Vec); +pub type ChunkIdentifier = (crate::SubtreePrefix, Option>, TreeType, Vec); pub const CURRENT_STATE_SYNC_VERSION: u16 = 1; @@ -82,7 +83,7 @@ impl GroveDb { } let root_app_hash = self.root_hash(transaction, grove_version).value?; - let (chunk_prefix, root_key, is_sum_tree, chunk_id) = + let (chunk_prefix, root_key, tree_type, chunk_id) = utils::decode_global_chunk_id(global_chunk_id, &root_app_hash)?; // TODO: Refactor this by writing fetch_chunk_inner (as only merk constructor @@ -92,7 +93,7 @@ impl GroveDb { .open_transactional_merk_by_prefix( chunk_prefix, root_key, - is_sum_tree, + tree_type, tx, None, grove_version, @@ -138,7 +139,7 @@ impl GroveDb { .open_non_transactional_merk_by_prefix( chunk_prefix, root_key, - is_sum_tree, + tree_type, None, grove_version, ) @@ -258,7 +259,7 @@ pub(crate) mod utils { ed::Encode, proofs::{Decoder, Op}, }; - + use grovedb_merk::merk::TreeType; use crate::{replication::ChunkIdentifier, Error}; /// Converts a path, represented as a slice of byte vectors (`&[Vec]`), @@ -324,7 +325,7 @@ pub(crate) mod utils { if global_chunk_id == app_hash { let root_chunk_prefix_key: crate::SubtreePrefix = [0u8; 32]; - return Ok((root_chunk_prefix_key, None, false, vec![])); + return Ok((root_chunk_prefix_key, None, TreeType::NormalTree, vec![])); } let (chunk_prefix_key, remaining) = global_chunk_id.split_at(chunk_prefix_length); @@ -350,6 +351,8 @@ pub(crate) mod utils { } let (is_sum_tree, chunk_id) = remaining.split_at(is_sum_tree_length); + let tree_type = is_sum_tree[0].try_into()?; + let subtree_prefix: crate::SubtreePrefix = chunk_prefix_key .try_into() .map_err(|_| Error::CorruptedData("unable to construct subtree".to_string()))?; @@ -358,11 +361,11 @@ pub(crate) mod utils { Ok(( subtree_prefix, Some(root_key.to_vec()), - is_sum_tree[0] != 0, + tree_type, chunk_id.to_vec(), )) } else { - Ok((subtree_prefix, None, is_sum_tree[0] != 0, chunk_id.to_vec())) + Ok((subtree_prefix, None, tree_type, chunk_id.to_vec())) } } @@ -381,7 +384,7 @@ pub(crate) mod utils { pub fn encode_global_chunk_id( subtree_prefix: [u8; blake3::OUT_LEN], root_key_opt: Option>, - is_sum_tree: bool, + tree_type: TreeType, chunk_id: Vec, ) -> Vec { let mut res = vec![]; @@ -395,11 +398,7 @@ pub(crate) mod utils { res.push(0u8); } - let mut is_sum_tree_v = 0u8; - if is_sum_tree { - is_sum_tree_v = 1u8; - } - res.push(is_sum_tree_v); + res.push(tree_type as u8); res.extend(chunk_id.to_vec()); diff --git a/grovedb/src/replication/state_sync_session.rs b/grovedb/src/replication/state_sync_session.rs index 59d933165..3c48506dc 100644 --- a/grovedb/src/replication/state_sync_session.rs +++ b/grovedb/src/replication/state_sync_session.rs @@ -9,6 +9,7 @@ use grovedb_merk::{ tree::{kv::ValueDefinedCostType, value_hash}, CryptoHash, Restorer, }; +use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbImmediateStorageContext, RocksDbStorage}, @@ -37,8 +38,8 @@ struct SubtreeStateSyncInfo<'db> { /// Tree root key root_key: Option>, - /// Is Sum tree? - is_sum_tree: bool, + /// The type of tree + tree_type: TreeType, /// Path of current tree current_path: Vec>, @@ -130,7 +131,7 @@ impl<'tx> SubtreeStateSyncInfo<'tx> { SubtreeStateSyncInfo { restorer, root_key: None, - is_sum_tree: false, + tree_type: TreeType::NormalTree, pending_chunks: Default::default(), current_path: vec![], num_processed_chunks: 0, @@ -246,14 +247,14 @@ impl<'db> MultiStateSyncSession<'db> { &*(tx as *const _) }; - if let Ok((merk, root_key, is_sum_tree)) = + if let Ok((merk, root_key, tree_type)) = db.open_merk_for_replication(path.clone(), transaction_ref, grove_version) { let restorer = Restorer::new(merk, hash, actual_hash); let mut sync_info = SubtreeStateSyncInfo::new(restorer); sync_info.pending_chunks.insert(vec![]); sync_info.root_key = root_key.clone(); - sync_info.is_sum_tree = is_sum_tree; + sync_info.tree_type = tree_type; sync_info.current_path = path.to_vec(); self.as_mut() .current_prefixes() @@ -261,7 +262,7 @@ impl<'db> MultiStateSyncSession<'db> { Ok(encode_global_chunk_id( chunk_prefix, root_key, - is_sum_tree, + tree_type, vec![], )) } else { @@ -367,7 +368,7 @@ impl<'db> MultiStateSyncSession<'db> { next_chunk_ids.push(encode_global_chunk_id( chunk_prefix, subtree_state_sync.root_key.clone(), - subtree_state_sync.is_sum_tree, + subtree_state_sync.tree_type, local_chunk_id.clone(), )); } diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index d5ec111ea..896346c56 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -3213,7 +3213,7 @@ mod tests { let storage = db.db.get_storage_context(EMPTY_PATH, None).unwrap(); let root_merk = Merk::open_base( storage, - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -3318,7 +3318,7 @@ mod tests { let subtree = Merk::open_layered_with_root_key( subtree_storage, Some(b"key3".to_vec()), - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -3367,7 +3367,7 @@ mod tests { let subtree = Merk::open_layered_with_root_key( subtree_storage, Some(b"key4".to_vec()), - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) @@ -3387,7 +3387,7 @@ mod tests { let subtree = Merk::open_layered_with_root_key( subtree_storage, Some(b"key3".to_vec()), - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index b9b624a44..62e14eb01 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -28,7 +28,7 @@ macro_rules! storage_context_with_parent_optional_tx { $transaction:ident, $storage:ident, $root_key:ident, - $is_sum_tree:ident, + $tree_type:ident, $grove_version:ident, { $($body:tt)* } ) => { @@ -54,24 +54,13 @@ macro_rules! storage_context_with_parent_optional_tx { ) }) ); - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), )).wrap_with_cost($cost); - } - } + }; + $($body)* } else { return Err(Error::CorruptedData( "path is empty".to_owned(), @@ -95,24 +84,13 @@ macro_rules! storage_context_with_parent_optional_tx { ) }) ); - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), )).wrap_with_cost($cost); - } - } + }; + $($body)* } else { return Err(Error::CorruptedData( "path is empty".to_owned(), @@ -134,7 +112,7 @@ macro_rules! storage_context_with_parent_optional_tx_internal_error { $transaction:ident, $storage:ident, $root_key:ident, - $is_sum_tree:ident, + $tree_type:ident, $grove_version:ident, { $($body:tt)* } ) => { @@ -162,24 +140,13 @@ macro_rules! storage_context_with_parent_optional_tx_internal_error { }).unwrap_add_cost(&mut $cost); match result { Ok(element) => { - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( - "parent is not a tree" - .to_owned(), - )).wrap_with_cost($cost); - } - } + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + return Err(Error::CorruptedData( + "parent is not a tree" + .to_owned(), + )).wrap_with_cost($cost); + }; + $($body)* }, Err(e) => Err(e), } @@ -210,24 +177,13 @@ macro_rules! storage_context_with_parent_optional_tx_internal_error { }).unwrap_add_cost(&mut $cost); match result { Ok(element) => { - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( - "parent is not a tree" - .to_owned(), - )).wrap_with_cost($cost); - } - } + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + return Err(Error::CorruptedData( + "parent is not a tree" + .to_owned(), + )).wrap_with_cost($cost); + }; + $($body)* }, Err(e) => Err(e), } @@ -293,7 +249,7 @@ macro_rules! merk_optional_tx { &mut $cost, ::grovedb_merk::Merk::open_base( storage.unwrap_add_cost(&mut $cost), - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), $grove_version, ).map(|merk_res| @@ -315,7 +271,7 @@ macro_rules! merk_optional_tx { $transaction, storage, root_key, - is_sum_tree, + tree_type, $grove_version, { #[allow(unused_mut)] @@ -324,7 +280,7 @@ macro_rules! merk_optional_tx { ::grovedb_merk::Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), $grove_version, ).map(|merk_res| @@ -367,7 +323,7 @@ macro_rules! merk_optional_tx_internal_error { &mut $cost, ::grovedb_merk::Merk::open_base( storage.unwrap_add_cost(&mut $cost), - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), $grove_version ).map(|merk_res| @@ -389,7 +345,7 @@ macro_rules! merk_optional_tx_internal_error { $transaction, storage, root_key, - is_sum_tree, + tree_type, $grove_version, { #[allow(unused_mut)] @@ -398,7 +354,7 @@ macro_rules! merk_optional_tx_internal_error { ::grovedb_merk::Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), $grove_version, ).map(|merk_res| @@ -438,7 +394,7 @@ macro_rules! merk_optional_tx_path_not_empty { $transaction, storage, root_key, - is_sum_tree, + tree_type, $grove_version, { #[allow(unused_mut)] @@ -447,7 +403,7 @@ macro_rules! merk_optional_tx_path_not_empty { ::grovedb_merk::Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree, + tree_type, Some(&Element::value_defined_cost_for_serialized_value), $grove_version, ).map(|merk_res| diff --git a/grovedb/src/visualize.rs b/grovedb/src/visualize.rs index 39cf3432b..0659db9a3 100644 --- a/grovedb/src/visualize.rs +++ b/grovedb/src/visualize.rs @@ -95,6 +95,28 @@ impl Visualize for Element { drawer = root_key.as_deref().visualize(drawer)?; drawer.write(format!(" {value}").as_bytes())?; + if let Some(f) = flags { + if !f.is_empty() { + drawer = f.visualize(drawer)?; + } + } + } + Element::BigSumTree(root_key, value, flags) => { + drawer.write(b"big_sum_tree: ")?; + drawer = root_key.as_deref().visualize(drawer)?; + drawer.write(format!(" {value}").as_bytes())?; + + if let Some(f) = flags { + if !f.is_empty() { + drawer = f.visualize(drawer)?; + } + } + } + Element::CountTree(root_key, value, flags) => { + drawer.write(b"count_tree: ")?; + drawer = root_key.as_deref().visualize(drawer)?; + drawer.write(format!(" {value}").as_bytes())?; + if let Some(f) = flags { if !f.is_empty() { drawer = f.visualize(drawer)?; diff --git a/merk/src/error.rs b/merk/src/error.rs index f9ba79c0c..15733bf8b 100644 --- a/merk/src/error.rs +++ b/merk/src/error.rs @@ -120,6 +120,9 @@ pub enum Error { #[error("big sum tree under normal sum tree error {0}")] BigSumTreeUnderNormalSumTree(String), + + #[error("unknown tree type {0}")] + UnknownTreeType(String), } impl From for Error { diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index e1dfdccd0..406da48b1 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -59,6 +59,7 @@ use grovedb_version::version::GroveVersion; use source::MerkSource; use crate::{ + TreeFeatureType, error::Error, merk::{defaults::ROOT_KEY_KEY, options::MerkOptions}, proofs::{ @@ -243,15 +244,49 @@ impl MerkType { } } } -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum TreeType { - NormalTree, - SumTree, - BigSumTree, - CountTree, + NormalTree = 0, + SumTree = 1, + BigSumTree = 2, + CountTree = 3, +} + +impl TryFrom for TreeType { + type Error = Error; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(TreeType::NormalTree), + 1 => Ok(TreeType::SumTree), + 2 => Ok(TreeType::BigSumTree), + 3 => Ok(TreeType::CountTree), + n => Err(Error::UnknownTreeType(format!("got {}, max is 3", n))), // Error handling + } + } +} + +impl fmt::Display for TreeType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match *self { + TreeType::NormalTree => "Normal Tree", + TreeType::SumTree => "Sum Tree", + TreeType::BigSumTree => "Big Sum Tree", + TreeType::CountTree => "Count Tree", + }; + write!(f, "{}", s) + } } impl TreeType { + pub fn allows_sum_item(&self) -> bool { + match self { + TreeType::NormalTree => false, + TreeType::SumTree => true, + TreeType::BigSumTree => true, + TreeType::CountTree => false, + } + } pub fn inner_node_type(&self) -> NodeType { match self { TreeType::NormalTree => NodeType::NormalNode, @@ -260,6 +295,15 @@ impl TreeType { TreeType::CountTree => NodeType::CountNode, } } + + pub fn empty_tree_feature_type(&self) -> TreeFeatureType { + match self { + TreeType::NormalTree => TreeFeatureType::BasicMerkNode, + TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), + TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), + TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), + } + } } #[cfg(any(feature = "full", feature = "verify"))] diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index f82f92aa3..ed9d470e2 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -94,6 +94,12 @@ impl TreeNodeInner { self.kv.value } + /// Get the value as owned of the key value struct + pub fn value_as_owned_with_feature(self) -> (Vec, TreeFeatureType) { + (self.kv.value, self.kv.feature_type) + } + + /// Get the value as slice of the key value struct pub fn value_as_slice(&self) -> &[u8] { self.kv.value.as_slice() From 1483e18d247d5190f65c9a665bb38cdce7b31627 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 8 Jan 2025 06:26:01 +0700 Subject: [PATCH 04/30] more work --- grovedb-version/src/version/v2.rs | 2 +- grovedb/Cargo.toml | 2 +- .../estimated_costs/average_case_costs.rs | 25 +-- grovedb/src/element/helpers.rs | 12 ++ .../src/estimated_costs/average_case_costs.rs | 196 ++++++++++-------- grovedb/src/operations/get/query.rs | 23 +- grovedb/src/util.rs | 2 +- 7 files changed, 153 insertions(+), 109 deletions(-) diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 5e8c58e7d..46ec8142d 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -144,7 +144,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { }, average_case: GroveDBOperationsAverageCaseVersions { add_average_case_get_merk_at_path: 0, - average_case_merk_replace_tree: 0, + average_case_merk_replace_tree: 1, //changed average_case_merk_insert_tree: 0, average_case_merk_delete_tree: 0, average_case_merk_insert_element: 0, diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index f84735c05..c294463dc 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -51,7 +51,7 @@ name = "insertion_benchmark" harness = false [features] -default = ["full"] +default = ["full", "estimated_costs"] proof_debug = ["grovedb-merk/proof_debug"] serde = ["dep:serde", "grovedb-merk/serde", "indexmap/serde"] full = [ diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 051025069..abd699302 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -21,7 +21,7 @@ use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; - +use grovedb_merk::merk::TreeType; use crate::Element; #[cfg(feature = "full")] use crate::{ @@ -44,7 +44,7 @@ impl GroveOp { propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { - let in_tree_using_sums = layer_element_estimates.is_sum_tree; + let in_tree_type = layer_element_estimates.tree_type; let propagate_if_input = || { if propagate { Some(layer_element_estimates) @@ -53,10 +53,10 @@ impl GroveOp { } }; match self { - GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::average_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, - sum.is_some(), + aggregate_data, propagate, grove_version, ), @@ -119,16 +119,9 @@ impl GroveOp { propagate, grove_version, ), - GroveOp::DeleteTree => GroveDb::average_case_merk_delete_tree( - key, - false, - layer_element_estimates, - propagate, - grove_version, - ), - GroveOp::DeleteSumTree => GroveDb::average_case_merk_delete_tree( + GroveOp::DeleteTree(tree_type) => GroveDb::average_case_merk_delete_tree( key, - true, + tree_type, layer_element_estimates, propagate, grove_version, @@ -142,7 +135,7 @@ impl GroveOp { #[derive(Default)] pub(in crate::batch) struct AverageCaseTreeCacheKnownPaths { paths: HashMap, - cached_merks: HashMap, + cached_merks: HashMap, } #[cfg(feature = "full")] @@ -167,7 +160,7 @@ impl fmt::Debug for AverageCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for AverageCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, tree_type: TreeType) -> CostResult<(), Error> { let mut average_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -175,7 +168,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { // empty at this point. // There is however a hash call that creates the prefix average_case_cost.hash_node_calls += 1; - self.cached_merks.insert(inserted_path, is_sum_tree); + self.cached_merks.insert(inserted_path, tree_type); Ok(()).wrap_with_cost(average_case_cost) } diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 79c9205c2..4b0ecc80d 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -146,6 +146,18 @@ impl Element { } } + #[cfg(any(feature = "full", feature = "verify"))] + /// Check if the element is a tree and return the flags and the tree type + pub fn tree_flags_and_type(&self) -> Option<(&Option, TreeType)> { + match self { + Element::Tree(_, flags) => Some((flags, TreeType::NormalTree)), + Element::SumTree(_, _, flags) => Some((flags, TreeType::SumTree)), + Element::BigSumTree(_, _, flags) => Some((flags, TreeType::BigSumTree)), + Element::CountTree(_, _, flags) => Some((flags, TreeType::CountTree)), + _ => None, + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a tree and return the tree type pub fn tree_type(&self) -> Option { diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 3bf1dbe82..32248516e 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -26,6 +26,7 @@ use crate::{ element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, Element, ElementFlags, Error, GroveDb, }; +use crate::element::CostSize; impl GroveDb { /// Add average case for getting a merk tree @@ -33,7 +34,7 @@ impl GroveDb { cost: &mut OperationCost, path: &KeyInfoPath, merk_should_be_empty: bool, - is_sum_tree: bool, + in_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -56,7 +57,7 @@ impl GroveDb { cost.storage_loaded_bytes += TreeNode::average_case_encoded_tree_size( key.max_length() as u32, HASH_LENGTH as u32, - is_sum_tree, + in_tree_type.inner_node_type(), ) as u64; } } @@ -69,19 +70,34 @@ impl GroveDb { pub(crate) fn average_case_merk_replace_tree( key: &KeyInfo, estimated_layer_information: &EstimatedLayerInformation, - _is_sum_tree: bool, + replacing_tree_type: TreeType, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { - check_grovedb_v0_with_cost!( - "average_case_merk_replace_tree", - grove_version - .grovedb_versions - .operations - .average_case - .average_case_merk_replace_tree - ); + match grove_version + .grovedb_versions + .operations + .average_case + .average_case_merk_replace_tree { + 0 => Self::average_case_merk_replace_tree_v0(key, estimated_layer_information, replacing_tree_type, propagate), + 1 => Self::average_case_merk_replace_tree_v1(key, estimated_layer_information, replacing_tree_type, propagate), + version => Err(Error::VersionError(GroveVersionError::UnknownVersionMismatch { + method: "average_case_merk_replace_tree".to_string(), + known_versions: vec![0, 1], + received: version, + })).wrap_with_cost(OperationCost::default()) + } + } + /// Add average case for insertion into merk + fn average_case_merk_replace_tree_v0( + key: &KeyInfo, + estimated_layer_information: &EstimatedLayerInformation, + _replacing_tree_type: TreeType, + propagate: bool, + ) -> CostResult<(), Error> { + // In v0 we used the estimated layer information tree type (which is the parent) in order + // to figure out the cost let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; let flags_size = cost_return_on_error_no_add!( @@ -93,17 +109,13 @@ impl GroveDb { ) .map(|f| f + f.required_space() as u32) .unwrap_or_default(); - let tree_cost_size = if estimated_layer_information.is_sum_tree { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost_size = estimated_layer_information.tree_type.cost_size(); //this was wrong let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_replace_layered( &mut cost, key_len, layer_extra_size, - estimated_layer_information.is_sum_tree, + estimated_layer_information.tree_type.inner_node_type(), ); if propagate { add_average_case_merk_propagate(&mut cost, estimated_layer_information) @@ -114,12 +126,47 @@ impl GroveDb { .wrap_with_cost(cost) } + /// Add average case for insertion into merk + fn average_case_merk_replace_tree_v1( + key: &KeyInfo, + estimated_layer_information: &EstimatedLayerInformation, + replacing_tree_type: TreeType, + propagate: bool, + ) -> CostResult<(), Error> { + let mut cost = OperationCost::default(); + let key_len = key.max_length() as u32; + let flags_size = cost_return_on_error_no_add!( + &cost, + estimated_layer_information + .estimated_layer_sizes + .layered_flags_size() + .map_err(Error::MerkError) + ) + .map(|f| f + f.required_space() as u32) + .unwrap_or_default(); + let tree_cost_size = replacing_tree_type.cost_size(); + let layer_extra_size = tree_cost_size + flags_size; + add_average_case_merk_replace_layered( + &mut cost, + key_len, + layer_extra_size, + estimated_layer_information.tree_type.inner_node_type(), + ); + if propagate { + add_average_case_merk_propagate(&mut cost, estimated_layer_information) + .map_err(Error::MerkError) + } else { + Ok(()) + } + .wrap_with_cost(cost) + } + /// Add average case for insertion into merk pub fn average_case_merk_insert_tree( key: &KeyInfo, flags: &Option, - is_sum_tree: bool, - in_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, propagate_if_input: Option<&EstimatedLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -138,13 +185,9 @@ impl GroveDb { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); - let tree_cost_size = if is_sum_tree { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost_size = tree_type.cost_size(); let value_len = tree_cost_size + flags_len; - add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_tree_using_sums); + add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_parent_tree_type); if let Some(input) = propagate_if_input { add_average_case_merk_propagate(&mut cost, input).map_err(Error::MerkError) } else { @@ -156,7 +199,7 @@ impl GroveDb { /// Add average case for insertion into merk pub fn average_case_merk_delete_tree( key: &KeyInfo, - is_sum_tree: bool, + tree_type: TreeType, estimated_layer_information: &EstimatedLayerInformation, propagate: bool, grove_version: &GroveVersion, @@ -181,11 +224,7 @@ impl GroveDb { ) .map(|f| f + f.required_space() as u32) .unwrap_or_default(); - let tree_cost_size = if is_sum_tree { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost_size = tree_type.cost_size(); let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_delete_layered(&mut cost, key_len, layer_extra_size); if propagate { @@ -218,27 +257,22 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; - match value { - Element::Tree(_, flags) | Element::SumTree(_, _, flags) => { - let flags_len = flags.as_ref().map_or(0, |flags| { - let flags_len = flags.len() as u32; - flags_len + flags_len.required_space() as u32 - }); - let tree_cost_size = if value.is_sum_tree() { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; - let value_len = tree_cost_size + flags_len; - add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_tree_type) - } - _ => add_cost_case_merk_insert( + if let Some((flags, tree_type)) = value.tree_flags_and_type() { + let flags_len = flags.as_ref().map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let tree_cost_size = tree_type.cost_size(); + let value_len = tree_cost_size + flags_len; + add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_tree_type) + } else { + add_cost_case_merk_insert( &mut cost, key_len, cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, in_tree_type, - ), - }; + ) + } if let Some(level) = propagate_for_level { add_average_case_merk_propagate(&mut cost, level).map_err(Error::MerkError) } else { @@ -253,7 +287,7 @@ impl GroveDb { pub fn average_case_merk_replace_element( key: &KeyInfo, value: &Element, - in_tree_using_sums: bool, + in_tree_type: TreeType, propagate_for_level: Option<&EstimatedLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -269,22 +303,18 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; match value { - Element::Tree(_, flags) | Element::SumTree(_, _, flags) => { + Element::Tree(_, flags) | Element::SumTree(_, _, flags) | Element::BigSumTree(_, _, flags) | Element::CountTree(_, _, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); - let tree_cost_size = if value.is_sum_tree() { - SUM_TREE_COST_SIZE - } else { - TREE_COST_SIZE - }; + let tree_cost_size = value.tree_type().unwrap().cost_size(); let value_len = tree_cost_size + flags_len; add_cost_case_merk_replace_layered( &mut cost, key_len, value_len, - in_tree_using_sums, + in_tree_type, ) } Element::Item(_, flags) | Element::SumItem(_, flags) => { @@ -303,14 +333,14 @@ impl GroveDb { &mut cost, key_len, value_len, - in_tree_using_sums, + in_tree_type, ) } _ => add_cost_case_merk_replace_same_size( &mut cost, key_len, cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, - in_tree_using_sums, + in_tree_type, ), }; if let Some(level) = propagate_for_level { @@ -328,7 +358,7 @@ impl GroveDb { key: &KeyInfo, value: &Element, change_in_bytes: i32, - in_tree_using_sums: bool, + in_tree_type: TreeType, propagate_for_level: Option<&EstimatedLayerInformation>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -359,7 +389,7 @@ impl GroveDb { key_len, value_len, change_in_bytes, - in_tree_using_sums, + in_tree_type, ) } _ => { @@ -416,7 +446,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -431,7 +461,7 @@ impl GroveDb { let value_size = TreeNode::average_case_encoded_tree_size( key.max_length() as u32, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ); cost.seek_count += 1; cost.storage_loaded_bytes += value_size as u64; @@ -445,8 +475,8 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_flags_size: u32, - is_sum_tree: bool, - in_parent_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -458,17 +488,13 @@ impl GroveDb { .add_average_case_has_raw_tree_cost ); - let estimated_element_size = if is_sum_tree { - SUM_TREE_COST_SIZE + estimated_flags_size - } else { - TREE_COST_SIZE + estimated_flags_size - }; + let estimated_element_size = tree_type.cost_size() + estimated_flags_size; Self::add_average_case_has_raw_cost::( cost, path, key, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, ) } @@ -479,7 +505,7 @@ impl GroveDb { _path: &KeyInfoPath, key: &KeyInfo, estimated_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -496,7 +522,7 @@ impl GroveDb { cost, key.max_length() as u32, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ) .map_err(Error::MerkError) } @@ -507,8 +533,8 @@ impl GroveDb { _path: &KeyInfoPath, key: &KeyInfo, estimated_flags_size: u32, - is_sum_tree: bool, - in_parent_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result<(), Error> { check_grovedb_v0!( @@ -520,17 +546,13 @@ impl GroveDb { .add_average_case_get_raw_tree_cost ); - let estimated_element_size = if is_sum_tree { - SUM_TREE_COST_SIZE + estimated_flags_size - } else { - TREE_COST_SIZE + estimated_flags_size - }; + let estimated_element_size = tree_type.cost_size() + estimated_flags_size; cost.seek_count += 1; add_average_case_get_merk_node( cost, key.max_length() as u32, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ) .map_err(Error::MerkError) } @@ -541,7 +563,7 @@ impl GroveDb { cost: &mut OperationCost, path: &KeyInfoPath, key: &KeyInfo, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, estimated_element_size: u32, estimated_references_sizes: Vec, grove_version: &GroveVersion, @@ -559,7 +581,7 @@ impl GroveDb { let value_size: u32 = TreeNode::average_case_encoded_tree_size( key.max_length() as u32, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type.inner_node_type(), ); cost.seek_count += 1 + estimated_references_sizes.len() as u32; cost.storage_loaded_bytes += value_size as u64 @@ -586,7 +608,7 @@ mod test { }; use grovedb_version::version::GroveVersion; use tempfile::TempDir; - + use grovedb_merk::merk::TreeType; use crate::{ batch::{key_info::KeyInfo::KnownKey, KeyInfoPath}, tests::{common::EMPTY_PATH, TEST_LEAF}, @@ -606,7 +628,7 @@ mod test { storage .get_storage_context(EMPTY_PATH, Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -626,7 +648,7 @@ mod test { // Reopen merk: this time, only root node is loaded to memory let merk = Merk::open_base( storage.get_storage_context(EMPTY_PATH, None).unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -650,7 +672,7 @@ mod test { // (this will be the max_element_size) let mut cost = OperationCost::default(); let key = KnownKey(8_u64.to_be_bytes().to_vec()); - add_average_case_get_merk_node(&mut cost, key.max_length() as u32, 60, false) + add_average_case_get_merk_node(&mut cost, key.max_length() as u32, 60, TreeType::NormalTree.inner_node_type()) .expect("expected to add cost"); assert_eq!(cost, node_result.cost); } @@ -716,7 +738,7 @@ mod test { &path, &key, elem.serialized_size(grove_version).expect("expected size") as u32, - false, + TreeType::NormalTree, GroveVersion::latest(), ) .expect("expected to add cost"); diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 81046dbfc..778942e05 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -23,6 +23,7 @@ use crate::{ reference_path::ReferencePathType, Element, Error, GroveDb, PathQuery, TransactionArg, }; +use crate::element::{BigSumValue, CountValue}; #[cfg(feature = "full")] #[derive(Debug, Eq, PartialEq, Clone)] @@ -32,6 +33,10 @@ pub enum QueryItemOrSumReturnType { ItemData(Vec), /// A sum item or a sum tree value SumValue(SumValue), + /// A big sum tree value + BigSumValue(BigSumValue), + /// A count value + CountValue(CountValue), } #[cfg(feature = "full")] @@ -222,7 +227,7 @@ where { )), } } - Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) => Ok(element), + Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Ok(element), Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -341,7 +346,7 @@ where { } Element::Item(item, _) => Ok(item), Element::SumItem(item, _) => Ok(item.encode_var_vec()), - Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidQuery( + Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items and references", )), } @@ -422,6 +427,12 @@ where { Element::SumTree(_, sum_value, _) => { Ok(QueryItemOrSumReturnType::SumValue(sum_value)) } + Element::BigSumTree(_, big_sum_value, _) => { + Ok(QueryItemOrSumReturnType::BigSumValue(big_sum_value)) + } + Element::CountTree(_, count_value, _) => { + Ok(QueryItemOrSumReturnType::CountValue(count_value)) + } _ => Err(Error::InvalidQuery( "the reference must result in an item", )), @@ -439,6 +450,12 @@ where { Element::SumTree(_, sum_value, _) => { Ok(QueryItemOrSumReturnType::SumValue(sum_value)) } + Element::BigSumTree(_, big_sum_value, _) => { + Ok(QueryItemOrSumReturnType::BigSumValue(big_sum_value)) + } + Element::CountTree(_, count_value, _) => { + Ok(QueryItemOrSumReturnType::CountValue(count_value)) + } Element::Tree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items, sum items, references and sum \ trees", @@ -520,7 +537,7 @@ where { } } Element::SumItem(item, _) => Ok(item), - Element::Tree(..) | Element::SumTree(..) | Element::Item(..) => { + Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) | Element::Item(..) => { Err(Error::InvalidQuery( "path_queries over sum items can only refer to sum items and \ references", diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index 62e14eb01..63d214e4f 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -445,7 +445,7 @@ macro_rules! root_merk_optional_tx { &mut $cost, ::grovedb_merk::Merk::open_base( storage.unwrap_add_cost(&mut $cost), - false, + TreeType::NormalTree, Some(&Element::value_defined_cost_for_serialized_value), $grove_version, ).map(|merk_res| From cac1b376af4170f656a506cca2efd15b0e3b765e Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 8 Jan 2025 09:40:52 +0700 Subject: [PATCH 05/30] fmt --- grovedb-version/src/version/merk_versions.rs | 11 +- grovedb-version/src/version/mod.rs | 8 +- grovedb-version/src/version/v1.rs | 8 +- grovedb-version/src/version/v2.rs | 10 +- grovedb/src/batch/batch_structure.rs | 6 +- .../estimated_costs/average_case_costs.rs | 93 +++--- .../batch/estimated_costs/worst_case_costs.rs | 46 +-- .../batch/just_in_time_reference_update.rs | 8 +- grovedb/src/batch/mod.rs | 124 ++++--- .../src/batch/single_deletion_cost_tests.rs | 3 +- grovedb/src/element/constructor.rs | 2 +- grovedb/src/element/delete.rs | 26 +- grovedb/src/element/get.rs | 53 +-- grovedb/src/element/helpers.rs | 59 ++-- grovedb/src/element/insert.rs | 27 +- grovedb/src/element/mod.rs | 5 +- grovedb/src/element/query.rs | 4 +- .../src/estimated_costs/average_case_costs.rs | 104 +++--- .../src/estimated_costs/worst_case_costs.rs | 16 +- grovedb/src/lib.rs | 36 ++- grovedb/src/operations/delete/average_case.rs | 15 +- grovedb/src/operations/delete/mod.rs | 5 +- grovedb/src/operations/delete/worst_case.rs | 34 +- grovedb/src/operations/get/average_case.rs | 29 +- grovedb/src/operations/get/query.rs | 29 +- grovedb/src/operations/get/worst_case.rs | 13 +- grovedb/src/operations/is_empty_tree.rs | 3 +- grovedb/src/operations/proof/verify.rs | 5 +- grovedb/src/replication.rs | 6 +- grovedb/src/replication/state_sync_session.rs | 2 +- grovedb/src/tests/sum_tree_tests.rs | 59 +++- grovedb/src/util.rs | 14 +- .../src/estimated_costs/average_case_costs.rs | 306 +++++++++++++++++- merk/src/estimated_costs/mod.rs | 2 +- merk/src/estimated_costs/worst_case_costs.rs | 2 +- merk/src/merk/apply.rs | 4 +- merk/src/merk/mod.rs | 61 ++-- merk/src/merk/open.rs | 7 +- merk/src/merk/restore.rs | 13 +- merk/src/merk/source.rs | 2 +- merk/src/proofs/tree.rs | 18 +- merk/src/test_utils/mod.rs | 2 +- merk/src/test_utils/temp_merk.rs | 3 +- merk/src/tree/encoding.rs | 6 +- merk/src/tree/kv.rs | 12 +- merk/src/tree/link.rs | 48 ++- merk/src/tree/mod.rs | 39 ++- merk/src/tree/ops.rs | 2 +- merk/src/tree/tree_feature_type.rs | 73 +++-- merk/src/tree/walk/mod.rs | 4 +- node-grove/src/converter.rs | 4 + 51 files changed, 1022 insertions(+), 449 deletions(-) diff --git a/grovedb-version/src/version/merk_versions.rs b/grovedb-version/src/version/merk_versions.rs index fac25f913..f92d90132 100644 --- a/grovedb-version/src/version/merk_versions.rs +++ b/grovedb-version/src/version/merk_versions.rs @@ -1,2 +1,11 @@ +use versioned_feature_core::FeatureVersion; + #[derive(Clone, Debug, Default)] -pub struct MerkVersions {} +pub struct MerkVersions { + pub average_case_costs: MerkAverageCaseCostsVersions, +} + +#[derive(Clone, Debug, Default)] +pub struct MerkAverageCaseCostsVersions { + pub add_average_case_merk_propagate: FeatureVersion, +} diff --git a/grovedb-version/src/version/mod.rs b/grovedb-version/src/version/mod.rs index 8f6268419..d795176c1 100644 --- a/grovedb-version/src/version/mod.rs +++ b/grovedb-version/src/version/mod.rs @@ -6,7 +6,7 @@ pub mod v2; pub use versioned_feature_core::*; use crate::version::{ - grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2 + grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2, }; #[derive(Clone, Debug, Default)] @@ -17,6 +17,12 @@ pub struct GroveVersion { } impl GroveVersion { + pub fn first<'a>() -> &'a Self { + GROVE_VERSIONS + .first() + .expect("expected to have a platform version") + } + pub fn latest<'a>() -> &'a Self { GROVE_VERSIONS .last() diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 97cfb38b3..71a1c71f3 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -8,7 +8,7 @@ use crate::version::{ GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBReplicationVersions, GroveDBVersions, }, - merk_versions::MerkVersions, + merk_versions::{MerkAverageCaseCostsVersions, MerkVersions}, GroveVersion, }; @@ -184,5 +184,9 @@ pub const GROVE_V1: GroveVersion = GroveVersion { apply_chunk: 0, }, }, - merk_versions: MerkVersions {}, + merk_versions: MerkVersions { + average_case_costs: MerkAverageCaseCostsVersions { + add_average_case_merk_propagate: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 46ec8142d..304b034c1 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -8,7 +8,7 @@ use crate::version::{ GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBReplicationVersions, GroveDBVersions, }, - merk_versions::MerkVersions, + merk_versions::{MerkAverageCaseCostsVersions, MerkVersions}, GroveVersion, }; @@ -144,7 +144,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { }, average_case: GroveDBOperationsAverageCaseVersions { add_average_case_get_merk_at_path: 0, - average_case_merk_replace_tree: 1, //changed + average_case_merk_replace_tree: 1, // changed average_case_merk_insert_tree: 0, average_case_merk_delete_tree: 0, average_case_merk_insert_element: 0, @@ -184,5 +184,9 @@ pub const GROVE_V2: GroveVersion = GroveVersion { apply_chunk: 0, }, }, - merk_versions: MerkVersions {}, + merk_versions: MerkVersions { + average_case_costs: MerkAverageCaseCostsVersions { + add_average_case_merk_propagate: 1, + }, + }, }; diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index bd50c796a..8ed802f99 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -129,9 +129,9 @@ where } Ok(()) } - GroveOp::RefreshReference { .. } - | GroveOp::Delete - | GroveOp::DeleteTree(_) => Ok(()), + GroveOp::RefreshReference { .. } | GroveOp::Delete | GroveOp::DeleteTree(_) => { + Ok(()) + } GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { Err(Error::InvalidBatchOperation( "replace and insert tree hash are internal operations only", diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index abd699302..93c6dccda 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -10,18 +10,18 @@ use std::{ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::RootHashKeyAndAggregateData; #[cfg(feature = "full")] use grovedb_merk::{ estimated_costs::average_case_costs::{average_case_merk_propagate, EstimatedLayerInformation}, IsSumTree, }; +use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use grovedb_merk::merk::TreeType; + use crate::Element; #[cfg(feature = "full")] use crate::{ @@ -53,28 +53,32 @@ impl GroveOp { } }; match self { - GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => GroveDb::average_case_merk_replace_tree( - key, - layer_element_estimates, - aggregate_data, - propagate, - grove_version, - ), - GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { - GroveDb::average_case_merk_insert_tree( + GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => { + GroveDb::average_case_merk_replace_tree( key, - flags, - sum.is_some(), - in_tree_using_sums, - propagate_if_input(), + layer_element_estimates, + aggregate_data.parent_tree_type(), + propagate, grove_version, ) } + GroveOp::InsertTreeWithRootHash { + flags, + aggregate_data, + .. + } => GroveDb::average_case_merk_insert_tree( + key, + flags, + aggregate_data.parent_tree_type(), + in_tree_type, + propagate_if_input(), + grove_version, + ), GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { GroveDb::average_case_merk_insert_element( key, element, - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ) @@ -91,14 +95,14 @@ impl GroveOp { *max_reference_hop, flags.clone(), ), - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ), GroveOp::Replace { element } => GroveDb::average_case_merk_replace_element( key, element, - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ), @@ -109,7 +113,7 @@ impl GroveOp { key, element, *change_in_bytes, - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ), @@ -121,7 +125,7 @@ impl GroveOp { ), GroveOp::DeleteTree(tree_type) => GroveDb::average_case_merk_delete_tree( key, - tree_type, + *tree_type, layer_element_estimates, propagate, grove_version, @@ -231,12 +235,11 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { &mut cost, path, layer_should_be_empty, - layer_info.is_sum_tree, + layer_info.tree_type, grove_version, ) ); - self.cached_merks - .insert(path.clone(), layer_info.is_sum_tree); + self.cached_merks.insert(path.clone(), layer_info.tree_type); } for (key, op) in ops_at_path_by_key.into_iter() { @@ -248,9 +251,10 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { cost_return_on_error!( &mut cost, - average_case_merk_propagate(layer_element_estimates).map_err(Error::MerkError) + average_case_merk_propagate(layer_element_estimates, grove_version) + .map_err(Error::MerkError) ); - Ok(([0u8; 32], None, None)).wrap_with_cost(cost) + Ok(([0u8; 32], None, AggregateData::NoAggregateData)).wrap_with_cost(cost) } fn update_base_merk_root_key( @@ -272,12 +276,12 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { estimated_layer_info .estimated_layer_count .estimated_to_be_empty(), - estimated_layer_info.is_sum_tree, + estimated_layer_info.tree_type, grove_version ) ); self.cached_merks - .insert(base_path, estimated_layer_info.is_sum_tree); + .insert(base_path, estimated_layer_info.tree_type); } } Ok(()).wrap_with_cost(cost) @@ -293,11 +297,14 @@ mod tests { storage_cost::{removal::StorageRemovedBytes::NoStorageRemoval, StorageCost}, OperationCost, }; - use grovedb_merk::estimated_costs::average_case_costs::{ - EstimatedLayerCount::{ApproximateElements, EstimatedLevel, PotentiallyAtMaxElements}, - EstimatedLayerInformation, - EstimatedLayerSizes::{AllItems, AllSubtrees}, - EstimatedSumTrees::{NoSumTrees, SomeSumTrees}, + use grovedb_merk::{ + estimated_costs::average_case_costs::{ + EstimatedLayerCount::{ApproximateElements, EstimatedLevel, PotentiallyAtMaxElements}, + EstimatedLayerInformation, + EstimatedLayerSizes::{AllItems, AllSubtrees}, + EstimatedSumTrees::{NoSumTrees, SomeSumTrees}, + }, + merk::TreeType, }; use grovedb_version::version::GroveVersion; @@ -325,7 +332,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: ApproximateElements(0), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -394,7 +401,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, Some(3)), }, @@ -402,7 +409,7 @@ mod tests { paths.insert( KeyInfoPath(vec![KeyInfo::KnownKey(b"key1".to_vec())]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -461,7 +468,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllItems(4, 3, None), }, @@ -534,7 +541,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(1, false), estimated_layer_sizes: AllSubtrees(1, NoSumTrees, None), }, @@ -620,7 +627,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, false), estimated_layer_sizes: AllSubtrees(1, NoSumTrees, None), }, @@ -629,7 +636,7 @@ mod tests { paths.insert( KeyInfoPath(vec![KeyInfo::KnownKey(b"0".to_vec())]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -700,7 +707,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(1, false), estimated_layer_sizes: AllSubtrees( 1, @@ -715,7 +722,7 @@ mod tests { paths.insert( KeyInfoPath::from_known_owned_path(vec![vec![7]]), EstimatedLayerInformation { - is_sum_tree: true, + tree_type: TreeType::SumTree, estimated_layer_count: PotentiallyAtMaxElements, estimated_layer_sizes: AllItems(32, 8, None), }, @@ -778,7 +785,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(1, false), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -787,7 +794,7 @@ mod tests { paths.insert( KeyInfoPath(vec![KeyInfo::KnownKey(b"0".to_vec())]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 0eb4efe06..b22368731 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -14,13 +14,13 @@ use grovedb_costs::{ use grovedb_merk::estimated_costs::worst_case_costs::{ worst_case_merk_propagate, WorstCaseLayerInformation, }; -use grovedb_merk::RootHashKeyAndAggregateData; +use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use grovedb_merk::merk::TreeType; + use crate::Element; #[cfg(feature = "full")] use crate::{ @@ -49,24 +49,28 @@ impl GroveOp { } }; match self { - GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => GroveDb::worst_case_merk_replace_tree( - key, - aggregate_data, - in_parent_tree_type, - worst_case_layer_element_estimates, - propagate, - grove_version, - ), - GroveOp::InsertTreeWithRootHash { flags, aggregate_data, .. } => { - GroveDb::worst_case_merk_insert_tree( + GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => { + GroveDb::worst_case_merk_replace_tree( key, - flags, - aggregate_data, + aggregate_data.parent_tree_type(), in_parent_tree_type, - propagate_if_input(), + worst_case_layer_element_estimates, + propagate, grove_version, ) } + GroveOp::InsertTreeWithRootHash { + flags, + aggregate_data, + .. + } => GroveDb::worst_case_merk_insert_tree( + key, + flags, + aggregate_data.parent_tree_type(), + in_parent_tree_type, + propagate_if_input(), + grove_version, + ), GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { GroveDb::worst_case_merk_insert_element( key, @@ -117,7 +121,7 @@ impl GroveOp { ), GroveOp::DeleteTree(tree_type) => GroveDb::worst_case_merk_delete_tree( key, - tree_type, + *tree_type, worst_case_layer_element_estimates, propagate, grove_version, @@ -156,7 +160,7 @@ impl fmt::Debug for WorstCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for WorstCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &QualifiedGroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, _tree_type: TreeType) -> CostResult<(), Error> { let mut worst_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -201,7 +205,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { GroveDb::add_worst_case_get_merk_at_path::( &mut cost, path, - false, + TreeType::NormalTree, grove_version, ) ); @@ -213,7 +217,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { &mut cost, op.worst_case_cost( &key, - false, + TreeType::NormalTree, worst_case_layer_element_estimates, false, grove_version @@ -225,7 +229,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { &mut cost, worst_case_merk_propagate(worst_case_layer_element_estimates).map_err(Error::MerkError) ); - Ok(([0u8; 32], None, None)).wrap_with_cost(cost) + Ok(([0u8; 32], None, AggregateData::NoAggregateData)).wrap_with_cost(cost) } fn update_base_merk_root_key( @@ -244,7 +248,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { GroveDb::add_worst_case_get_merk_at_path::( &mut cost, &base_path, - false, + TreeType::NormalTree, grove_version, ) ); diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index 9fb1ec158..e0fd5f94c 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -9,10 +9,10 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ + merk::TreeType, tree::{kv::KV, value_hash, TreeNode}, CryptoHash, Merk, }; -use grovedb_merk::merk::TreeType; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; @@ -103,7 +103,11 @@ where in_tree_type.inner_node_type(), ) } else { - KV::node_value_byte_cost_size(key.len() as u32, serialized.len() as u32, in_tree_type.inner_node_type()) + KV::node_value_byte_cost_size( + key.len() as u32, + serialized.len() as u32, + in_tree_type.inner_node_type(), + ) }; let mut i = 0; diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 87bd59f62..97b0ed721 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -48,12 +48,12 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ + merk::TreeType, tree::{ kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, - value_hash, NULL_HASH, + value_hash, AggregateData, NULL_HASH, }, CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndAggregateData, - TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; use grovedb_path::SubtreePath; use grovedb_storage::{ @@ -66,8 +66,6 @@ use grovedb_version::{ use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; -use grovedb_merk::merk::{NodeType, TreeType}; -use grovedb_merk::tree::AggregateData; use key_info::{KeyInfo, KeyInfo::KnownKey}; pub use options::BatchApplyOptions; @@ -76,7 +74,10 @@ pub use crate::batch::batch_structure::{OpsByLevelPath, OpsByPath}; use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, - element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, TREE_COST_SIZE}, + element::{ + MaxReferenceHop, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, + SUM_TREE_COST_SIZE, TREE_COST_SIZE, + }, operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, @@ -854,10 +855,10 @@ where /// /// # Returns /// - /// * `Ok((Element, Vec, TreeType))` - Returns the deserialized `Element` - /// and the serialized counterpart if the retrieval and deserialization - /// are successful, wrapped in the associated cost. Also returns if the - /// merk of the element is a sum tree as a TreeType. + /// * `Ok((Element, Vec, TreeType))` - Returns the deserialized + /// `Element` and the serialized counterpart if the retrieval and + /// deserialization are successful, wrapped in the associated cost. Also + /// returns if the merk of the element is a sum tree as a TreeType. /// * `Err(Error)` - Returns an error if any issue occurs during the /// retrieval or deserialization of the referenced element. /// @@ -1018,7 +1019,10 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1136,12 +1140,13 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { - Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost) - } + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), } } GroveOp::InsertOnly { element } => match element { @@ -1165,7 +1170,10 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1191,12 +1199,10 @@ where grove_version, ) } - GroveOp::Delete | GroveOp::DeleteTree(_) => { - Err(Error::InvalidBatchOperation( - "references can not point to something currently being deleted", - )) - .wrap_with_cost(cost) - } + GroveOp::Delete | GroveOp::DeleteTree(_) => Err(Error::InvalidBatchOperation( + "references can not point to something currently being deleted", + )) + .wrap_with_cost(cost), } } else { self.process_reference( @@ -1339,7 +1345,10 @@ where ) ); } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element @@ -1489,7 +1498,7 @@ where key_info.get_key(), false, in_tree_type, /* we are in a sum tree, this might or might not be a - * sum item */ + * sum item */ &mut batch_operations, grove_version ) @@ -1533,14 +1542,26 @@ where aggregate_data, } => { let element = match aggregate_data { - AggregateData::NoAggregateData => Element::new_tree_with_flags(root_key, flags), - AggregateData::Sum(sum_value) => Element::new_sum_tree_with_flags_and_sum_value( - root_key, aggregate_data.as_i64(), flags, - ), - AggregateData::BigSum(sum_value) => Element::new_big_sum_tree_with_flags_and_sum_value( - root_key, aggregate_data.as_i128(), flags, - ), - AggregateData::Count(count_value) => Element::new_count_tree_with_flags_and_count_value(root_key, aggregate_data.as_u64(), flags) + AggregateData::NoAggregateData => { + Element::new_tree_with_flags(root_key, flags) + } + AggregateData::Sum(sum_value) => { + Element::new_sum_tree_with_flags_and_sum_value( + root_key, sum_value, flags, + ) + } + AggregateData::BigSum(sum_value) => { + Element::new_big_sum_tree_with_flags_and_sum_value( + root_key, sum_value, flags, + ) + } + AggregateData::Count(count_value) => { + Element::new_count_tree_with_flags_and_count_value( + root_key, + count_value, + flags, + ) + } }; let merk_feature_type = cost_return_on_error_no_add!(&cost, element.get_feature_type(in_tree_type)); @@ -1569,8 +1590,13 @@ where &[], Some(batch_apply_options.as_merk_options()), &|key, value| { - Element::specialized_costs_for_key_value(key, value, in_tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Element::specialized_costs_for_key_value( + key, + value, + in_tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), &|old_value, new_value| { @@ -1621,7 +1647,10 @@ where // we need to give back the value defined cost in the case that the // new element is a tree match new_element { - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let tree_type = new_element.tree_type().unwrap(); let tree_cost_size = match tree_type { TreeType::NormalTree => TREE_COST_SIZE, @@ -1820,7 +1849,8 @@ impl GroveDb { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), - aggregate_data: AggregateData::NoAggregateData, + aggregate_data: + AggregateData::NoAggregateData, } .into(); } else if let Element::SumTree(.., flags) = @@ -1848,8 +1878,7 @@ impl GroveDb { )) .wrap_with_cost(cost); } - GroveOp::Delete - | GroveOp::DeleteTree(_) => { + GroveOp::Delete | GroveOp::DeleteTree(_) => { if calculated_root_key.is_some() { return Err(Error::InvalidBatchOperation( "modification of tree when it will be \ @@ -2140,7 +2169,12 @@ impl GroveDb { if let Some((parent_path, parent_key)) = path.derive_parent() { if new_merk { // TODO: can this be a sum tree - Ok(Merk::open_empty(storage, MerkType::LayeredMerk, TreeType::NormalTree)).wrap_with_cost(cost) + Ok(Merk::open_empty( + storage, + MerkType::LayeredMerk, + TreeType::NormalTree, + )) + .wrap_with_cost(cost) } else { let parent_storage = self .db @@ -2183,7 +2217,12 @@ impl GroveDb { } } } else if new_merk { - Ok(Merk::open_empty(storage, MerkType::BaseMerk, TreeType::NormalTree)).wrap_with_cost(cost) + Ok(Merk::open_empty( + storage, + MerkType::BaseMerk, + TreeType::NormalTree, + )) + .wrap_with_cost(cost) } else { Merk::open_base( storage, @@ -2223,7 +2262,8 @@ impl GroveDb { } else { MerkType::LayeredMerk }; - Ok(Merk::open_empty(storage, merk_type, TreeType::NormalTree)).wrap_with_cost(local_cost) + Ok(Merk::open_empty(storage, merk_type, TreeType::NormalTree)) + .wrap_with_cost(local_cost) } else if let Some((base_path, last)) = path.derive_parent() { let parent_storage = self .db diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index fceccce6e..ee5c6902b 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -7,9 +7,10 @@ mod tests { Identifier, StorageRemovalPerEpochByIdentifier, StorageRemovedBytes::SectionedStorageRemoval, }; + use grovedb_merk::merk::TreeType; use grovedb_version::version::GroveVersion; use intmap::IntMap; - use grovedb_merk::merk::TreeType; + use crate::{ batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 3cc381a62..556df1dac 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -1,13 +1,13 @@ //! Constructor //! Functions for setting an element's type +use crate::element::{BigSumValue, CountValue}; #[cfg(feature = "full")] use crate::{ element::{MaxReferenceHop, SumValue}, reference_path::ReferencePathType, Element, ElementFlags, }; -use crate::element::{BigSumValue, CountValue}; impl Element { #[cfg(feature = "full")] diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index af2992f1f..2ab01e02f 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -5,9 +5,9 @@ use grovedb_costs::OperationCost; #[cfg(feature = "full")] use grovedb_costs::{storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt}; +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_merk::{BatchEntry, Error as MerkError, Merk, MerkOptions, Op}; -use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_storage::StorageContext; #[cfg(feature = "full")] @@ -43,14 +43,20 @@ impl Element { | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; - let tree_type = merk.tree_type; //todo not sure we get it again, we need to see if this is necessary + // todo not sure we get it again, we need to see if this is necessary + let tree_type = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch, &[], merk_options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, @@ -94,14 +100,20 @@ impl Element { | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; - let tree_type = merk.tree_type; //todo not sure we get it again, we need to see if this is necessary + // todo not sure we get it again, we need to see if this is necessary + let tree_type = merk.tree_type; merk.apply_with_costs_just_in_time_value_update::<_, Vec>( &batch, &[], merk_options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), &|_, _| Ok(None), diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 876971a79..f61a20d61 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -5,19 +5,19 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::tree::kv::KV; #[cfg(feature = "full")] use grovedb_merk::Merk; #[cfg(feature = "full")] use grovedb_merk::{ed::Decode, tree::TreeNodeInner}; +use grovedb_merk::{merk::NodeType, tree::kv::KV}; #[cfg(feature = "full")] use grovedb_storage::StorageContext; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; -use grovedb_merk::merk::NodeType; -use crate::element::{CostSize, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}; + +use crate::element::{CostSize, SUM_ITEM_COST_SIZE}; #[cfg(feature = "full")] use crate::{Element, Error, Hash}; @@ -121,14 +121,18 @@ impl Element { match grove_version .grovedb_versions .element - .get_optional_from_storage { + .get_optional_from_storage + { 0 => Self::get_optional_from_storage_v0(storage, key, grove_version), 1 => Self::get_optional_from_storage_v1(storage, key, grove_version), - version => Err(Error::VersionError(GroveVersionError::UnknownVersionMismatch { - method: "get_optional_from_storage".to_string(), - known_versions: vec![0, 1], - received: version, - })).wrap_with_cost(OperationCost::default()) + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "get_optional_from_storage".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )) + .wrap_with_cost(OperationCost::default()), } } @@ -187,10 +191,16 @@ impl Element { flags_len + flags_len.required_space() as u32 }); let value_len = cost_size + flags_len; - cost.storage_loaded_bytes = - KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, NodeType::NormalNode) as u64 + cost.storage_loaded_bytes = KV::node_value_byte_cost_size( + key_ref.len() as u32, + value_len, + NodeType::NormalNode, + ) as u64 } - Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags )) | Some(Element::BigSumTree(_, _, flags )) | Some(Element::CountTree(_, _, flags )) => { + Some(Element::Tree(_, flags)) + | Some(Element::SumTree(_, _, flags)) + | Some(Element::BigSumTree(_, _, flags)) + | Some(Element::CountTree(_, _, flags)) => { let tree_cost_size = element.as_ref().unwrap().tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -209,7 +219,6 @@ impl Element { Ok(element).wrap_with_cost(cost) } - #[cfg(feature = "full")] /// Get an element directly from storage under a key /// Merk does not need to be loaded @@ -236,15 +245,17 @@ impl Element { .transpose() ); - let Some((value, tree_feature_type)) = maybe_tree_inner.map(|tree_inner| tree_inner.value_as_owned_with_feature()) else { - return Ok(None).wrap_with_cost(cost) + let Some((value, tree_feature_type)) = + maybe_tree_inner.map(|tree_inner| tree_inner.value_as_owned_with_feature()) + else { + return Ok(None).wrap_with_cost(cost); }; let node_type = tree_feature_type.node_type(); let element = cost_return_on_error_no_add!( &cost, Self::deserialize(value.as_slice(), grove_version).map_err(|_| { - Error::CorruptedData(String::from("unable to deserialize element")) - }) + Error::CorruptedData(String::from("unable to deserialize element")) + }) ); match &element { Element::Item(..) | Element::Reference(..) => { @@ -264,9 +275,13 @@ impl Element { }); let value_len = cost_size + flags_len; cost.storage_loaded_bytes = - KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, node_type) as u64 // this is changed to sum node in v1 + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, node_type) as u64 + // this is changed to sum node in v1 } - Element::Tree(_, flags) | Element::SumTree(_, _, flags ) | Element::BigSumTree(_, _, flags ) | Element::CountTree(_, _, flags ) => { + Element::Tree(_, flags) + | Element::SumTree(_, _, flags) + | Element::BigSumTree(_, _, flags) + | Element::CountTree(_, _, flags) => { let tree_cost_size = element.tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 4b0ecc80d..f22a2c7e4 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -6,6 +6,10 @@ use grovedb_merk::tree::kv::{ ValueDefinedCostType, ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, }; +use grovedb_merk::{ + merk::{NodeType, TreeType}, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, +}; #[cfg(feature = "full")] use grovedb_merk::{ tree::{kv::KV, TreeNode}, @@ -16,8 +20,8 @@ use grovedb_merk::{ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; -use grovedb_merk::merk::{NodeType, TreeType}; -use grovedb_merk::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; + +use crate::element::BIG_SUM_TREE_COST_SIZE; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; #[cfg(any(feature = "full", feature = "verify"))] @@ -29,7 +33,6 @@ use crate::{ }; #[cfg(any(feature = "full", feature = "verify"))] use crate::{Element, Error}; -use crate::element::BIG_SUM_TREE_COST_SIZE; impl Element { #[cfg(any(feature = "full", feature = "verify"))] @@ -47,7 +50,9 @@ impl Element { /// everything else pub fn big_sum_value_or_default(&self) -> i128 { match self { - Element::SumItem(sum_value, _) | Element::SumTree(_, sum_value, _) => *sum_value as i128, + Element::SumItem(sum_value, _) | Element::SumTree(_, sum_value, _) => { + *sum_value as i128 + } Element::BigSumTree(_, sum_value, _) => *sum_value, _ => 0, } @@ -123,25 +128,27 @@ impl Element { } #[cfg(any(feature = "full", feature = "verify"))] - /// Check if the element is a tree and return the root_tree info and tree type + /// Check if the element is a tree and return the root_tree info and tree + /// type pub fn root_key_and_tree_type_owned(self) -> Option<(Option>, TreeType)> { match self { Element::Tree(root_key, _) => Some((root_key, TreeType::NormalTree)), - Element::SumTree(root_key, _, _) => Some((root_key, TreeType::SumTree)), - Element::BigSumTree(root_key, _, _) => Some((root_key, TreeType::BigSumTree)), - Element::CountTree(root_key, _, _) => Some((root_key, TreeType::CountTree)), + Element::SumTree(root_key, ..) => Some((root_key, TreeType::SumTree)), + Element::BigSumTree(root_key, ..) => Some((root_key, TreeType::BigSumTree)), + Element::CountTree(root_key, ..) => Some((root_key, TreeType::CountTree)), _ => None, } } #[cfg(any(feature = "full", feature = "verify"))] - /// Check if the element is a tree and return the root_tree info and the tree type + /// Check if the element is a tree and return the root_tree info and the + /// tree type pub fn root_key_and_tree_type(&self) -> Option<(&Option>, TreeType)> { match self { Element::Tree(root_key, _) => Some((root_key, TreeType::NormalTree)), - Element::SumTree(root_key, _, _) => Some((root_key, TreeType::SumTree)), - Element::BigSumTree(root_key, _, _) => Some((root_key, TreeType::BigSumTree)), - Element::CountTree(root_key, _, _) => Some((root_key, TreeType::CountTree)), + Element::SumTree(root_key, ..) => Some((root_key, TreeType::SumTree)), + Element::BigSumTree(root_key, ..) => Some((root_key, TreeType::BigSumTree)), + Element::CountTree(root_key, ..) => Some((root_key, TreeType::CountTree)), _ => None, } } @@ -162,10 +169,10 @@ impl Element { /// Check if the element is a tree and return the tree type pub fn tree_type(&self) -> Option { match self { - Element::Tree(_, _) => Some(TreeType::NormalTree), - Element::SumTree(_, _, _) => Some(TreeType::SumTree), - Element::BigSumTree(_, _, _) => Some(TreeType::BigSumTree), - Element::CountTree(_, _, _) => Some(TreeType::CountTree), + Element::Tree(..) => Some(TreeType::NormalTree), + Element::SumTree(..) => Some(TreeType::SumTree), + Element::BigSumTree(..) => Some(TreeType::BigSumTree), + Element::CountTree(..) => Some(TreeType::CountTree), _ => None, } } @@ -185,7 +192,13 @@ impl Element { #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a tree pub fn is_any_tree(&self) -> bool { - matches!(self, Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..)) + matches!( + self, + Element::SumTree(..) + | Element::Tree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) + ) } #[cfg(any(feature = "full", feature = "verify"))] @@ -350,9 +363,7 @@ impl Element { let value_len = TREE_COST_SIZE + flags_len; let key_len = key.len() as u32; KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - node_type, + key_len, value_len, node_type, ) } Element::SumTree(_, _sum_value, flags) => { @@ -363,9 +374,7 @@ impl Element { let value_len = SUM_TREE_COST_SIZE + flags_len; let key_len = key.len() as u32; KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - node_type, + key_len, value_len, node_type, ) } Element::BigSumTree(_, _sum_value, flags) => { @@ -376,9 +385,7 @@ impl Element { let value_len = BIG_SUM_TREE_COST_SIZE + flags_len; let key_len = key.len() as u32; KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - node_type, + key_len, value_len, node_type, ) } Element::SumItem(.., flags) => { diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 2dd56da5f..f32a1f358 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -62,8 +62,13 @@ impl Element { options, &|key, value| { // it is possible that a normal item was being replaced with a - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, @@ -320,8 +325,13 @@ impl Element { &[], options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, @@ -407,8 +417,13 @@ impl Element { &[], options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index b09844588..2784bb98f 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -27,7 +27,9 @@ use bincode::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; #[cfg(feature = "full")] -use grovedb_merk::estimated_costs::{LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, BIG_SUM_LAYER_COST_SIZE, BIG_SUM_VALUE_EXTRA_COST}; +use grovedb_merk::estimated_costs::{ + BIG_SUM_LAYER_COST_SIZE, LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, +}; #[cfg(feature = "full")] use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] @@ -97,7 +99,6 @@ impl CostSize for TreeType { } } - #[cfg(any(feature = "full", feature = "verify"))] /// Variants of GroveDB stored entities /// diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 5a94797c4..0df089719 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -9,6 +9,8 @@ use grovedb_costs::{ OperationCost, }; #[cfg(feature = "full")] +use grovedb_merk::merk::TreeType; +#[cfg(feature = "full")] use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(feature = "full")] use grovedb_merk::proofs::query::SubqueryBranch; @@ -22,8 +24,6 @@ use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, StorageConte use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; -#[cfg(feature = "full")] -use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 32248516e..ed07857ee 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -12,6 +12,7 @@ use grovedb_merk::{ add_average_case_merk_replace_layered, EstimatedLayerInformation, }, }, + merk::TreeType, tree::TreeNode, HASH_LENGTH, }; @@ -20,13 +21,12 @@ use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; -use grovedb_merk::merk::TreeType; + use crate::{ batch::{key_info::KeyInfo, KeyInfoPath}, - element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, + element::{CostSize, SUM_ITEM_COST_SIZE}, Element, ElementFlags, Error, GroveDb, }; -use crate::element::CostSize; impl GroveDb { /// Add average case for getting a merk tree @@ -78,14 +78,30 @@ impl GroveDb { .grovedb_versions .operations .average_case - .average_case_merk_replace_tree { - 0 => Self::average_case_merk_replace_tree_v0(key, estimated_layer_information, replacing_tree_type, propagate), - 1 => Self::average_case_merk_replace_tree_v1(key, estimated_layer_information, replacing_tree_type, propagate), - version => Err(Error::VersionError(GroveVersionError::UnknownVersionMismatch { - method: "average_case_merk_replace_tree".to_string(), - known_versions: vec![0, 1], - received: version, - })).wrap_with_cost(OperationCost::default()) + .average_case_merk_replace_tree + { + 0 => Self::average_case_merk_replace_tree_v0( + key, + estimated_layer_information, + replacing_tree_type, + propagate, + grove_version, + ), + 1 => Self::average_case_merk_replace_tree_v1( + key, + estimated_layer_information, + replacing_tree_type, + propagate, + grove_version, + ), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "average_case_merk_replace_tree".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )) + .wrap_with_cost(OperationCost::default()), } } @@ -95,9 +111,10 @@ impl GroveDb { estimated_layer_information: &EstimatedLayerInformation, _replacing_tree_type: TreeType, propagate: bool, + grove_version: &GroveVersion, ) -> CostResult<(), Error> { - // In v0 we used the estimated layer information tree type (which is the parent) in order - // to figure out the cost + // In v0 we used the estimated layer information tree type (which is the parent) + // in order to figure out the cost let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; let flags_size = cost_return_on_error_no_add!( @@ -109,7 +126,7 @@ impl GroveDb { ) .map(|f| f + f.required_space() as u32) .unwrap_or_default(); - let tree_cost_size = estimated_layer_information.tree_type.cost_size(); //this was wrong + let tree_cost_size = estimated_layer_information.tree_type.cost_size(); // this was wrong let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_replace_layered( &mut cost, @@ -118,7 +135,7 @@ impl GroveDb { estimated_layer_information.tree_type.inner_node_type(), ); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) @@ -132,6 +149,7 @@ impl GroveDb { estimated_layer_information: &EstimatedLayerInformation, replacing_tree_type: TreeType, propagate: bool, + grove_version: &GroveVersion, ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; @@ -142,8 +160,8 @@ impl GroveDb { .layered_flags_size() .map_err(Error::MerkError) ) - .map(|f| f + f.required_space() as u32) - .unwrap_or_default(); + .map(|f| f + f.required_space() as u32) + .unwrap_or_default(); let tree_cost_size = replacing_tree_type.cost_size(); let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_replace_layered( @@ -153,12 +171,12 @@ impl GroveDb { estimated_layer_information.tree_type.inner_node_type(), ); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) } - .wrap_with_cost(cost) + .wrap_with_cost(cost) } /// Add average case for insertion into merk @@ -189,7 +207,8 @@ impl GroveDb { let value_len = tree_cost_size + flags_len; add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_parent_tree_type); if let Some(input) = propagate_if_input { - add_average_case_merk_propagate(&mut cost, input).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, input, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -228,7 +247,7 @@ impl GroveDb { let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_delete_layered(&mut cost, key_len, layer_extra_size); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) @@ -274,7 +293,8 @@ impl GroveDb { ) } if let Some(level) = propagate_for_level { - add_average_case_merk_propagate(&mut cost, level).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, level, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -303,19 +323,17 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; match value { - Element::Tree(_, flags) | Element::SumTree(_, _, flags) | Element::BigSumTree(_, _, flags) | Element::CountTree(_, _, flags) => { + Element::Tree(_, flags) + | Element::SumTree(_, _, flags) + | Element::BigSumTree(_, _, flags) + | Element::CountTree(_, _, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); let tree_cost_size = value.tree_type().unwrap().cost_size(); let value_len = tree_cost_size + flags_len; - add_cost_case_merk_replace_layered( - &mut cost, - key_len, - value_len, - in_tree_type, - ) + add_cost_case_merk_replace_layered(&mut cost, key_len, value_len, in_tree_type) } Element::Item(_, flags) | Element::SumItem(_, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { @@ -329,12 +347,7 @@ impl GroveDb { cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32 }; let value_len = sum_item_cost_size + flags_len; - add_cost_case_merk_replace_same_size( - &mut cost, - key_len, - value_len, - in_tree_type, - ) + add_cost_case_merk_replace_same_size(&mut cost, key_len, value_len, in_tree_type) } _ => add_cost_case_merk_replace_same_size( &mut cost, @@ -344,7 +357,8 @@ impl GroveDb { ), }; if let Some(level) = propagate_for_level { - add_average_case_merk_propagate(&mut cost, level).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, level, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -398,7 +412,8 @@ impl GroveDb { } }; if let Some(level) = propagate_for_level { - add_average_case_merk_propagate(&mut cost, level).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, level, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -432,7 +447,7 @@ impl GroveDb { ); add_average_case_merk_delete(&mut cost, key_len, value_size); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) @@ -600,7 +615,7 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ - estimated_costs::average_case_costs::add_average_case_get_merk_node, + estimated_costs::average_case_costs::add_average_case_get_merk_node, merk::TreeType, test_utils::make_batch_seq, tree::kv::ValueDefinedCostType, Merk, }; use grovedb_storage::{ @@ -608,7 +623,7 @@ mod test { }; use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use grovedb_merk::merk::TreeType; + use crate::{ batch::{key_info::KeyInfo::KnownKey, KeyInfoPath}, tests::{common::EMPTY_PATH, TEST_LEAF}, @@ -672,8 +687,13 @@ mod test { // (this will be the max_element_size) let mut cost = OperationCost::default(); let key = KnownKey(8_u64.to_be_bytes().to_vec()); - add_average_case_get_merk_node(&mut cost, key.max_length() as u32, 60, TreeType::NormalTree.inner_node_type()) - .expect("expected to add cost"); + add_average_case_get_merk_node( + &mut cost, + key.max_length() as u32, + 60, + TreeType::NormalTree.inner_node_type(), + ) + .expect("expected to add cost"); assert_eq!(cost, node_result.cost); } diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index d745669dc..54cabcc42 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -14,6 +14,7 @@ use grovedb_merk::{ MERK_BIGGEST_VALUE_SIZE, }, }, + merk::TreeType, tree::TreeNode, HASH_LENGTH, }; @@ -22,13 +23,12 @@ use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; -use grovedb_merk::merk::TreeType; + use crate::{ batch::{key_info::KeyInfo, KeyInfoPath}, - element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, + element::{CostSize, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, Element, ElementFlags, Error, GroveDb, }; -use crate::element::CostSize; pub const WORST_CASE_FLAGS_LEN: u32 = 16386; // 2 bytes to represent this number for varint @@ -56,7 +56,7 @@ impl GroveDb { cost.storage_loaded_bytes += TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, HASH_LENGTH as u32, - tree_type.inner_node_type(), //todo This is probably wrong + tree_type.inner_node_type(), // todo This is probably wrong ) as u64; } } @@ -189,7 +189,10 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; match value { - Element::Tree(_, flags) | Element::SumTree(_, _, flags) | Element::BigSumTree(_, _, flags) | Element::CountTree(_, _, flags) => { + Element::Tree(_, flags) + | Element::SumTree(_, _, flags) + | Element::BigSumTree(_, _, flags) + | Element::CountTree(_, _, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 @@ -494,6 +497,7 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ estimated_costs::worst_case_costs::add_worst_case_get_merk_node, + merk::{NodeType, TreeType}, test_utils::{empty_path_merk, empty_path_merk_read_only, make_batch_seq}, tree::kv::ValueDefinedCostType, }; @@ -504,7 +508,7 @@ mod test { }; use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use grovedb_merk::merk::{NodeType, TreeType}; + use crate::{ batch::{key_info::KeyInfo::KnownKey, KeyInfoPath}, tests::{common::EMPTY_PATH, TEST_LEAF}, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index f89bd9c2a..84cfeac53 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -186,6 +186,7 @@ use grovedb_merk::{ tree::{combine_hash, value_hash}, BatchEntry, CryptoHash, KVIterator, Merk, }; +use grovedb_merk::{merk::TreeType, tree::AggregateData}; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] @@ -209,8 +210,7 @@ pub use query::{PathQuery, SizedQuery}; use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] use tokio::net::ToSocketAddrs; -use grovedb_merk::merk::TreeType; -use grovedb_merk::tree::AggregateData; + #[cfg(feature = "full")] use crate::element::helpers::raw_decode; #[cfg(any(feature = "full", feature = "verify"))] @@ -422,7 +422,10 @@ impl GroveDb { }) .unwrap()?; let tree_type = element.tree_type(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) | Element::BigSumTree(root_key, ..) = element { + if let Element::Tree(root_key, _) + | Element::SumTree(root_key, ..) + | Element::BigSumTree(root_key, ..) = element + { let tree_type = tree_type.expect("expected tree type"); Ok(( Merk::open_layered_with_root_key( @@ -496,7 +499,10 @@ impl GroveDb { ) ); let tree_type = element.tree_type(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) | Element::BigSumTree(root_key, ..) = element { + if let Element::Tree(root_key, _) + | Element::SumTree(root_key, ..) + | Element::BigSumTree(root_key, ..) = element + { let tree_type = tree_type.expect("expected tree type"); Merk::open_layered_with_root_key( storage, @@ -655,7 +661,9 @@ impl GroveDb { ); let (root_hash, root_key, aggregate_data) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) + child_tree + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -709,7 +717,9 @@ impl GroveDb { ); let (root_hash, root_key, sum) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) + child_tree + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -760,7 +770,9 @@ impl GroveDb { ); let (root_hash, root_key, sum) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) + child_tree + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -1129,7 +1141,10 @@ impl GroveDb { while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; match element { - Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::SumTree(..) + | Element::Tree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -1273,7 +1288,10 @@ impl GroveDb { while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; match element { - Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::SumTree(..) + | Element::Tree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 986f2b90a..1b018dbfa 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -8,6 +8,7 @@ use grovedb_merk::{ average_case_costs::EstimatedLayerInformation, worst_case_costs::add_average_case_cost_for_is_empty_tree_except, }, + merk::TreeType, HASH_LENGTH_U32, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; @@ -65,7 +66,7 @@ impl GroveDb { except_keys_count, key_len, estimated_element_size, - is_sum_tree, + tree_type, ) = cost_return_on_error_no_add!( &cost, if height == path_len - 1 { @@ -84,7 +85,7 @@ impl GroveDb { 0, key.max_length() as u32, estimated_value_len, - layer_info.is_sum_tree, + layer_info.tree_type, )) } else { Err(Error::InvalidParameter( @@ -109,7 +110,7 @@ impl GroveDb { 1, last_key.max_length() as u32, estimated_value_len, - layer_info.is_sum_tree, + layer_info.tree_type, )) } else { Err(Error::InvalidParameter("intermediate layer info missing")) @@ -121,7 +122,7 @@ impl GroveDb { Self::average_case_delete_operation_for_delete::( &KeyInfoPath::from_vec(path_at_level.to_vec()), key_at_level, - is_sum_tree, + tree_type, validate, check_if_tree, except_keys_count, @@ -139,7 +140,7 @@ impl GroveDb { pub fn average_case_delete_operation_for_delete<'db, S: Storage<'db>>( path: &KeyInfoPath, key: &KeyInfo, - parent_tree_is_sum_tree: bool, + in_parent_tree_type: TreeType, validate: bool, check_if_tree: bool, except_keys_count: u16, @@ -163,7 +164,7 @@ impl GroveDb { &mut cost, path, false, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); @@ -176,7 +177,7 @@ impl GroveDb { path, key, estimated_key_element_size.1, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index ec8ef98fb..01d2252f0 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -18,10 +18,9 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{proofs::Query, KVIterator}; +use grovedb_merk::{merk::TreeType, proofs::Query, KVIterator}; #[cfg(feature = "full")] use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; -use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_storage::{ @@ -592,7 +591,7 @@ impl GroveDb { // If there is any current batch operation that is inserting something in this // tree then it is not empty either is_empty &= !current_batch_operations.iter().any(|op| match op.op { - GroveOp::Delete | GroveOp::DeleteTree(_) => false, + GroveOp::Delete | GroveOp::DeleteTree(_) => false, // todo: fix for to_path (it clones) _ => op.path.to_path() == subtree_merk_path_vec, }); diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 8533cde55..09f9da76c 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -4,7 +4,8 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ - estimated_costs::worst_case_costs::add_worst_case_cost_for_is_empty_tree_except, tree::kv::KV, + estimated_costs::worst_case_costs::add_worst_case_cost_for_is_empty_tree_except, + merk::TreeType, tree::kv::KV, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; use grovedb_version::{ @@ -26,7 +27,7 @@ impl GroveDb { key: &KeyInfo, stop_path_height: Option, validate: bool, - intermediate_tree_info: IntMap<(bool, u32)>, + intermediate_tree_info: IntMap<(TreeType, u32)>, max_element_size: u32, grove_version: &GroveVersion, ) -> CostResult, Error> { @@ -59,13 +60,12 @@ impl GroveDb { check_if_tree, except_keys_count, max_element_size, - is_sum_tree, + tree_type, ) = cost_return_on_error_no_add!( &cost, if height == path_len { - if let Some((is_in_sum_tree, _)) = intermediate_tree_info.get(height as u64) - { - Ok((used_path, key, true, 0, max_element_size, *is_in_sum_tree)) + if let Some((tree_type, _)) = intermediate_tree_info.get(height as u64) { + Ok((used_path, key, true, 0, max_element_size, *tree_type)) } else { Err(Error::InvalidParameter( "intermediate flag size missing for height at path length", @@ -74,25 +74,19 @@ impl GroveDb { } else { let (last_key, smaller_path) = used_path.split_last().unwrap(); used_path = smaller_path; - if let Some((is_in_sum_tree, flags_size_at_level)) = + if let Some((tree_type, flags_size_at_level)) = intermediate_tree_info.get(height as u64) { // the worst case is that we are only in sum trees + // Todo the worst case is actually now big sum trees let value_len = SUM_TREE_COST_SIZE + flags_size_at_level; let max_tree_size = KV::layered_node_byte_cost_size_for_key_and_value_lengths( last_key.max_length() as u32, value_len, - *is_in_sum_tree, + tree_type.inner_node_type(), ); - Ok(( - used_path, - last_key, - false, - 1, - max_tree_size, - *is_in_sum_tree, - )) + Ok((used_path, last_key, false, 1, max_tree_size, *tree_type)) } else { Err(Error::InvalidParameter("intermediate flag size missing")) } @@ -103,7 +97,7 @@ impl GroveDb { Self::worst_case_delete_operation_for_delete::( &KeyInfoPath::from_vec(path_at_level.to_vec()), key_at_level, - is_sum_tree, + tree_type, validate, check_if_tree, except_keys_count, @@ -121,7 +115,7 @@ impl GroveDb { pub fn worst_case_delete_operation_for_delete<'db, S: Storage<'db>>( path: &KeyInfoPath, key: &KeyInfo, - parent_tree_is_sum_tree: bool, + in_parent_tree_type: TreeType, validate: bool, check_if_tree: bool, except_keys_count: u16, @@ -144,7 +138,7 @@ impl GroveDb { GroveDb::add_worst_case_get_merk_at_path::( &mut cost, path, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); @@ -157,7 +151,7 @@ impl GroveDb { path, key, max_element_size, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); diff --git a/grovedb/src/operations/get/average_case.rs b/grovedb/src/operations/get/average_case.rs index aca4426df..e05e63b1d 100644 --- a/grovedb/src/operations/get/average_case.rs +++ b/grovedb/src/operations/get/average_case.rs @@ -2,6 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::OperationCost; +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -21,7 +22,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -38,7 +39,7 @@ impl GroveDb { path, key, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -50,8 +51,8 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_flags_size: u32, - is_sum_tree: bool, - in_parent_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -68,8 +69,8 @@ impl GroveDb { path, key, estimated_flags_size, - is_sum_tree, - in_parent_tree_using_sums, + tree_type, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -81,7 +82,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -98,7 +99,7 @@ impl GroveDb { path, key, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -108,7 +109,7 @@ impl GroveDb { pub fn average_case_for_get( path: &KeyInfoPath, key: &KeyInfo, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, estimated_element_size: u32, estimated_references_sizes: Vec, grove_version: &GroveVersion, @@ -126,7 +127,7 @@ impl GroveDb { &mut cost, path, key, - in_parent_tree_using_sums, + in_parent_tree_type, estimated_element_size, estimated_references_sizes, grove_version, @@ -139,8 +140,8 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_flags_size: u32, - is_sum_tree: bool, - in_parent_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -157,8 +158,8 @@ impl GroveDb { path, key, estimated_flags_size, - is_sum_tree, - in_parent_tree_using_sums, + tree_type, + in_parent_tree_type, grove_version, )?; Ok(cost) diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 778942e05..14ed383b6 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -14,7 +14,8 @@ use integer_encoding::VarInt; #[cfg(feature = "full")] use crate::element::SumValue; use crate::{ - element::QueryOptions, operations::proof::ProveOptions, + element::{BigSumValue, CountValue, QueryOptions}, + operations::proof::ProveOptions, query_result_type::PathKeyOptionalElementTrio, }; #[cfg(feature = "full")] @@ -23,7 +24,6 @@ use crate::{ reference_path::ReferencePathType, Element, Error, GroveDb, PathQuery, TransactionArg, }; -use crate::element::{BigSumValue, CountValue}; #[cfg(feature = "full")] #[derive(Debug, Eq, PartialEq, Clone)] @@ -227,7 +227,11 @@ where { )), } } - Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Ok(element), + Element::Item(..) + | Element::SumItem(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Ok(element), Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -346,7 +350,10 @@ where { } Element::Item(item, _) => Ok(item), Element::SumItem(item, _) => Ok(item.encode_var_vec()), - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidQuery( + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items and references", )), } @@ -537,12 +544,14 @@ where { } } Element::SumItem(item, _) => Ok(item), - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) | Element::Item(..) => { - Err(Error::InvalidQuery( - "path_queries over sum items can only refer to sum items and \ - references", - )) - } + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) + | Element::Item(..) => Err(Error::InvalidQuery( + "path_queries over sum items can only refer to sum items and \ + references", + )), } } _ => Err(Error::CorruptedCodeExecution( diff --git a/grovedb/src/operations/get/worst_case.rs b/grovedb/src/operations/get/worst_case.rs index 7554a9111..663e74bbf 100644 --- a/grovedb/src/operations/get/worst_case.rs +++ b/grovedb/src/operations/get/worst_case.rs @@ -2,6 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::OperationCost; +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -20,7 +21,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, max_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -37,7 +38,7 @@ impl GroveDb { path, key, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -48,7 +49,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, max_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -65,7 +66,7 @@ impl GroveDb { path, key, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -77,7 +78,7 @@ impl GroveDb { key: &KeyInfo, max_element_size: u32, max_references_sizes: Vec, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -94,7 +95,7 @@ impl GroveDb { path, key, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, max_references_sizes, grove_version, )?; diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 9548286e4..37a56d643 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -2,11 +2,12 @@ #[cfg(feature = "full")] use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_version::error::GroveVersionError; use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; -use grovedb_merk::merk::TreeType; + #[cfg(feature = "full")] use crate::{util::merk_optional_tx, Element, Error, GroveDb, TransactionArg}; diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 7ff21910a..0130bac62 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -308,7 +308,10 @@ impl GroveDb { println!("lower layer had key {}", hex_to_ascii(key)); } match element { - Element::Tree(Some(_), _) | Element::SumTree(Some(_), ..) | Element::BigSumTree(Some(_), ..) | Element::CountTree(Some(_), ..) => { + Element::Tree(Some(_), _) + | Element::SumTree(Some(_), ..) + | Element::BigSumTree(Some(_), ..) + | Element::CountTree(Some(_), ..) => { path.push(key); let lower_hash = Self::verify_layer_proof( lower_layer, diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 9bb559c56..5f167719c 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -2,8 +2,7 @@ mod state_sync_session; use std::pin::Pin; -use grovedb_merk::{tree::hash::CryptoHash, ChunkProducer}; -use grovedb_merk::merk::TreeType; +use grovedb_merk::{merk::TreeType, tree::hash::CryptoHash, ChunkProducer}; use grovedb_path::SubtreePath; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -257,9 +256,10 @@ impl GroveDb { pub(crate) mod utils { use grovedb_merk::{ ed::Encode, + merk::TreeType, proofs::{Decoder, Op}, }; - use grovedb_merk::merk::TreeType; + use crate::{replication::ChunkIdentifier, Error}; /// Converts a path, represented as a slice of byte vectors (`&[Vec]`), diff --git a/grovedb/src/replication/state_sync_session.rs b/grovedb/src/replication/state_sync_session.rs index 3c48506dc..79defcfde 100644 --- a/grovedb/src/replication/state_sync_session.rs +++ b/grovedb/src/replication/state_sync_session.rs @@ -6,10 +6,10 @@ use std::{ }; use grovedb_merk::{ + merk::TreeType, tree::{kv::ValueDefinedCostType, value_hash}, CryptoHash, Restorer, }; -use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbImmediateStorageContext, RocksDbStorage}, diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index d72c26f76..1371e27e5 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -2,10 +2,9 @@ use grovedb_merk::{ proofs::Query, - tree::kv::ValueDefinedCostType, + tree::{kv::ValueDefinedCostType, AggregateData}, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; -use grovedb_merk::tree::AggregateData; use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; @@ -319,7 +318,10 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("node should exist"), Some(SummedMerkNode(0)) )); - assert_eq!(merk.aggregate_data().expect("expected to get sum").as_i64(), 40); + assert_eq!( + merk.aggregate_data().expect("expected to get sum").as_i64(), + 40 + ); // Perform the same test on regular trees let db = make_test_grovedb(grove_version); @@ -384,7 +386,10 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("node should exist"), Some(BasicMerkNode) )); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::NoAggregateData); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::NoAggregateData + ); } #[test] @@ -414,7 +419,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::NoAggregateData); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::NoAggregateData + ); // Add sum tree db.insert( @@ -453,7 +461,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(30)); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(30) + ); // Add more sum items db.insert( @@ -484,7 +495,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(70)); // 30 - 10 + 50 = 70 + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(70) + ); // 30 - 10 + 50 = 70 // Add non sum items, result should remain the same db.insert( @@ -505,7 +519,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(70)); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(70) + ); // Update existing sum items db.insert( @@ -536,7 +553,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(-60)); // 30 + 10 - 100 = -60 + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(-60) + ); // 30 + 10 - 100 = -60 // We can not replace a normal item with a sum item, so let's delete it first db.delete( @@ -567,10 +587,13 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(9999940)); // 30 + - // 10 - - // 100 + - // 10000000 + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(9999940) + ); // 30 + + // 10 - + // 100 + + // 10000000 // TODO: Test out overflows } @@ -866,7 +889,10 @@ fn test_sum_tree_with_batches() { .expect("node should exist"), Some(SummedMerkNode(10)) )); - assert_eq!(sum_tree.aggregate_data().expect("expected to get sum"), AggregateData::Sum(20)); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(20) + ); // Test propagation // Add a new sum tree with its own sum items, should affect sum of original @@ -941,5 +967,8 @@ fn test_sum_tree_with_batches() { ) .unwrap() .expect("should open tree"); - assert_eq!(sum_tree.aggregate_data().expect("expected to get sum"), AggregateData::Sum(41)); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(41) + ); } diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index 63d214e4f..20ec46d8a 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -54,7 +54,8 @@ macro_rules! storage_context_with_parent_optional_tx { ) }) ); - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), @@ -84,7 +85,8 @@ macro_rules! storage_context_with_parent_optional_tx { ) }) ); - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), @@ -140,7 +142,9 @@ macro_rules! storage_context_with_parent_optional_tx_internal_error { }).unwrap_add_cost(&mut $cost); match result { Ok(element) => { - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) + = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), @@ -177,7 +181,9 @@ macro_rules! storage_context_with_parent_optional_tx_internal_error { }).unwrap_add_cost(&mut $cost); match result { Ok(element) => { - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) + = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 89f7b94b9..f5a8fdd37 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -2,9 +2,14 @@ #[cfg(feature = "full")] use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::{ + error::GroveVersionError, + version::{FeatureVersion, GroveVersion}, +}; #[cfg(feature = "full")] use integer_encoding::VarInt; +use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] use crate::{ error::Error, @@ -12,7 +17,6 @@ use crate::{ tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, HASH_LENGTH_U32, }; -use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] /// Average key size @@ -395,9 +399,12 @@ pub fn add_average_case_merk_root_hash(cost: &mut OperationCost) { #[cfg(feature = "full")] /// Average case cost of propagating a merk -pub fn average_case_merk_propagate(input: &EstimatedLayerInformation) -> CostResult<(), Error> { +pub fn average_case_merk_propagate( + input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> CostResult<(), Error> { let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, input).wrap_with_cost(cost) + add_average_case_merk_propagate(&mut cost, input, grove_version).wrap_with_cost(cost) } #[cfg(feature = "full")] @@ -405,6 +412,297 @@ pub fn average_case_merk_propagate(input: &EstimatedLayerInformation) -> CostRes pub fn add_average_case_merk_propagate( cost: &mut OperationCost, input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> Result<(), Error> { + match grove_version + .merk_versions + .average_case_costs + .add_average_case_merk_propagate + { + 0 => add_average_case_merk_propagate_v0(cost, input), + 1 => add_average_case_merk_propagate_v1(cost, input), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "add_average_case_merk_propagate".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )), + } +} +#[cfg(feature = "full")] +/// Add average case cost for propagating a merk +fn add_average_case_merk_propagate_v1( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, +) -> Result<(), Error> { + let mut nodes_updated = 0; + // Propagation requires to recompute and write hashes up to the root + let EstimatedLayerInformation { + tree_type, + estimated_layer_count, + estimated_layer_sizes, + } = input; + let levels = estimated_layer_count.estimate_levels(); + nodes_updated += levels; + + if levels > 1 { + // we can get about 1 rotation, if there are more than 2 levels + nodes_updated += 1; + } + cost.seek_count += nodes_updated as u32; + + cost.hash_node_calls += nodes_updated * 2; + + cost.storage_cost.replaced_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + // it is normal to have LAYER_COST_SIZE here, as we add estimated sum tree + // additions right after + let value_len = LAYER_COST_SIZE + + average_flags_size + .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); + // in order to simplify calculations we get the estimated size and remove the + // cost for the basic merk + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + nodes_updated + * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = match subtrees_size { + None => 0, + Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition; + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates"))? + } + }; + let item_node_updates_cost = match items_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let reference_node_updates_cost = match references_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_replaced_bytes = total_updates_cost / weighted_nodes_updated; + if total_replaced_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_replaced_bytes as u32 + } + } + }; + cost.storage_loaded_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + nodes_updated + * KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = subtrees_size + .as_ref() + .map( + |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_node_updates_cost = items_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let reference_node_updates_cost = references_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + TreeType::NormalTree.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_loaded_bytes = total_updates_cost / weighted_nodes_updated; + if total_loaded_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_loaded_bytes as u32 + } + } + } as u64; + Ok(()) +} + +#[cfg(feature = "full")] +/// Add average case cost for propagating a merk +fn add_average_case_merk_propagate_v0( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, ) -> Result<(), Error> { let mut nodes_updated = 0; // Propagation requires to recompute and write hashes up to the root @@ -642,7 +940,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, value_len, - tree_type.inner_node_type(), //todo this is an error that we will need to fix however most likely references were never in sum trees + tree_type.inner_node_type(), // this was changed in v1 ); (*weight as u64) .checked_mul(cost as u64) diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index 6d0f47a67..5d90f0f63 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -5,9 +5,9 @@ use grovedb_costs::OperationCost; #[cfg(feature = "full")] use integer_encoding::VarInt; +use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] use crate::{tree::kv::KV, HASH_BLOCK_SIZE_U32, HASH_LENGTH_U32}; -use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] pub mod average_case_costs; diff --git a/merk/src/estimated_costs/worst_case_costs.rs b/merk/src/estimated_costs/worst_case_costs.rs index 0f206700a..b02753b82 100644 --- a/merk/src/estimated_costs/worst_case_costs.rs +++ b/merk/src/estimated_costs/worst_case_costs.rs @@ -33,6 +33,7 @@ use std::cmp::Ordering; #[cfg(feature = "full")] use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use crate::merk::NodeType; #[cfg(feature = "full")] use crate::{ error::Error, @@ -40,7 +41,6 @@ use crate::{ tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, }; -use crate::merk::NodeType; #[cfg(feature = "full")] #[derive(Clone, PartialEq, Eq, Debug)] diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 795124926..e524bac6e 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -11,13 +11,13 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ + merk::NodeType, tree::{ kv::{ValueDefinedCostType, KV}, AuxMerkBatch, Walker, }, Error, Merk, MerkBatch, MerkOptions, }; -use crate::merk::NodeType; impl<'db, S> Merk where @@ -65,7 +65,7 @@ where KB: AsRef<[u8]>, KA: AsRef<[u8]>, { - let node_type : NodeType = self.tree_type.inner_node_type(); + let node_type: NodeType = self.tree_type.inner_node_type(); self.apply_with_costs_just_in_time_value_update( batch, aux, diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 406da48b1..eadbe6966 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -59,7 +59,6 @@ use grovedb_version::version::GroveVersion; use source::MerkSource; use crate::{ - TreeFeatureType, error::Error, merk::{defaults::ROOT_KEY_KEY, options::MerkOptions}, proofs::{ @@ -71,13 +70,14 @@ use crate::{ Query, }, tree::{ - kv::ValueDefinedCostType, AuxMerkBatch, CryptoHash, Op, RefWalker, TreeNode, NULL_HASH, + kv::ValueDefinedCostType, AggregateData, AuxMerkBatch, CryptoHash, Op, RefWalker, TreeNode, + NULL_HASH, }, Error::{CostsError, EdError, StorageError}, Link, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, + TreeFeatureType, }; -use crate::tree::AggregateData; /// Key update types pub struct KeyUpdates { @@ -287,6 +287,7 @@ impl TreeType { TreeType::CountTree => false, } } + pub fn inner_node_type(&self) -> NodeType { match self { TreeType::NormalTree => NodeType::NormalNode, @@ -298,11 +299,11 @@ impl TreeType { pub fn empty_tree_feature_type(&self) -> TreeFeatureType { match self { - TreeType::NormalTree => TreeFeatureType::BasicMerkNode, - TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), - TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), - TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), - } + TreeType::NormalTree => TreeFeatureType::BasicMerkNode, + TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), + TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), + TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), + } } } @@ -407,9 +408,12 @@ where } /// Returns the root hash and non-prefixed key of the tree. - pub fn root_hash_key_and_aggregate_data(&self) -> CostResult { + pub fn root_hash_key_and_aggregate_data( + &self, + ) -> CostResult { self.use_tree(|tree| match tree { - None => Ok((NULL_HASH, None, AggregateData::NoAggregateData)).wrap_with_cost(Default::default()), + None => Ok((NULL_HASH, None, AggregateData::NoAggregateData)) + .wrap_with_cost(Default::default()), Some(tree) => { let aggregate_data = cost_return_on_error_default!(tree.aggregate_data()); tree.hash() @@ -756,9 +760,12 @@ where grove_version: &GroveVersion, ) { let (hash, key, aggregate_data) = match link { - Link::Reference { hash, key, aggregate_data, .. } => { - (hash.to_owned(), key.to_owned(), aggregate_data.to_owned()) - } + Link::Reference { + hash, + key, + aggregate_data, + .. + } => (hash.to_owned(), key.to_owned(), aggregate_data.to_owned()), Link::Modified { tree, .. } => ( tree.hash().unwrap(), tree.key().to_vec(), @@ -769,7 +776,11 @@ where child_heights: _, aggregate_data, tree, - } => (hash.to_owned(), tree.key().to_vec(), aggregate_data.to_owned()), + } => ( + hash.to_owned(), + tree.key().to_vec(), + aggregate_data.to_owned(), + ), _ => todo!(), }; @@ -852,7 +863,7 @@ mod test { use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use super::{Merk, RefWalker}; + use super::{Merk, RefWalker, TreeType}; use crate::{ merk::source::MerkSource, test_utils::*, tree::kv::ValueDefinedCostType, Op, TreeFeatureType::BasicMerkNode, @@ -1083,7 +1094,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1109,7 +1120,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1165,7 +1176,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1184,7 +1195,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1205,7 +1216,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1245,7 +1256,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1266,7 +1277,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1281,7 +1292,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1305,7 +1316,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1369,7 +1380,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index d47febcd4..fa172bf24 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -5,11 +5,11 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ + merk::TreeType, tree::kv::ValueDefinedCostType, Error, Merk, MerkType, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, }; -use crate::merk::TreeType; impl<'db, S> Merk where @@ -102,8 +102,9 @@ mod test { use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use crate::{tree::kv::ValueDefinedCostType, Merk, Op, TreeFeatureType::BasicMerkNode}; - use crate::merk::TreeType; + use crate::{ + merk::TreeType, tree::kv::ValueDefinedCostType, Merk, Op, TreeFeatureType::BasicMerkNode, + }; #[test] fn test_reopen_root_hash() { diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index cb694b418..5fb91b8be 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -36,7 +36,7 @@ use grovedb_version::version::GroveVersion; use crate::{ merk, - merk::MerkSource, + merk::{MerkSource, TreeType}, proofs::{ chunk::{ chunk::{LEFT, RIGHT}, @@ -52,7 +52,6 @@ use crate::{ Error::{CostsError, StorageError}, Link, Merk, }; -use crate::merk::TreeType; /// Restorer handles verification of chunks and replication of Merk trees. /// Chunks can be processed randomly as long as their parent has been processed @@ -318,7 +317,12 @@ impl<'db, S: StorageContext<'db>> Restorer { let updated_key = chunk_tree.key(); let updated_sum = chunk_tree.aggregate_data(); - if let Some(Link::Reference { key, aggregate_data, .. }) = parent.link_mut(*is_left) { + if let Some(Link::Reference { + key, + aggregate_data, + .. + }) = parent.link_mut(*is_left) + { *key = updated_key.to_vec(); *aggregate_data = updated_sum; } @@ -557,7 +561,7 @@ mod tests { use super::*; use crate::{ - merk::chunks::ChunkProducer, + merk::{chunks::ChunkProducer, TreeType}, proofs::chunk::{ chunk::tests::traverse_get_node_hash, error::ChunkError::InvalidChunkProof, }, @@ -565,7 +569,6 @@ mod tests { Error::ChunkRestoringError, Merk, PanicSource, }; - use crate::merk::TreeType; #[test] fn test_chunk_verification_non_avl_tree() { diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs index 620373a86..14c707958 100644 --- a/merk/src/merk/source.rs +++ b/merk/src/merk/source.rs @@ -3,10 +3,10 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ + merk::TreeType, tree::{kv::ValueDefinedCostType, Fetch, TreeNode}, Error, Link, Merk, }; -use crate::merk::TreeType; impl<'db, S> Merk where diff --git a/merk/src/proofs/tree.rs b/merk/src/proofs/tree.rs index 094fe48fa..6a1a93e53 100644 --- a/merk/src/proofs/tree.rs +++ b/merk/src/proofs/tree.rs @@ -21,8 +21,10 @@ use crate::{ Link, TreeFeatureType::SummedMerkNode, }; -use crate::tree::AggregateData; -use crate::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; +use crate::{ + tree::AggregateData, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, +}; #[cfg(any(feature = "full", feature = "verify"))] /// Contains a tree's child node and its hash. The hash can always be assumed to @@ -39,7 +41,9 @@ impl Child { #[cfg(feature = "full")] pub fn as_link(&self) -> Link { let (key, aggregate_data) = match &self.tree.node { - Node::KV(key, _) | Node::KVValueHash(key, ..) => (key.as_slice(), AggregateData::NoAggregateData), + Node::KV(key, _) | Node::KVValueHash(key, ..) => { + (key.as_slice(), AggregateData::NoAggregateData) + } Node::KVValueHashFeatureType(key, _, _, feature_type) => { (key.as_slice(), (*feature_type).into()) } @@ -634,7 +638,7 @@ mod test { left_link, Link::Reference { hash: tree.left.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (0, 0), key: vec![1] } @@ -644,7 +648,7 @@ mod test { right_link, Link::Reference { hash: tree.right.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (0, 0), key: vec![3] } @@ -683,7 +687,7 @@ mod test { left_link, Link::Reference { hash: tree.left.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: Some(3), + aggregate_data: AggregateData::Sum(3), child_heights: (0, 0), key: vec![1] } @@ -693,7 +697,7 @@ mod test { right_link, Link::Reference { hash: tree.right.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: Some(1), + aggregate_data: AggregateData::Sum(1), child_heights: (0, 0), key: vec![3] } diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 24ac94fad..f53ce730c 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -40,6 +40,7 @@ use rand::prelude::*; pub use temp_merk::TempMerk; use crate::{ + merk::TreeType, tree::{ kv::{ValueDefinedCostType, KV}, BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, TreeNode, Walker, @@ -47,7 +48,6 @@ use crate::{ Merk, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; -use crate::merk::TreeType; /// Assert tree invariants pub fn assert_tree_invariants(tree: &TreeNode) { diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 3133ce6bb..93cff86f5 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -40,10 +40,9 @@ use grovedb_storage::{ }; use grovedb_version::version::GroveVersion; -use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::Merk; -use crate::merk::TreeType; +use crate::{merk::TreeType, tree::kv::ValueDefinedCostType}; #[cfg(feature = "full")] /// Wraps a Merk instance and deletes it from disk it once it goes out of scope. diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index 31ef980e9..2f5b88c61 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -146,9 +146,11 @@ impl TreeNode { #[cfg(feature = "full")] #[cfg(test)] mod tests { - use crate::tree::AggregateData; use super::{super::Link, *}; - use crate::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; + use crate::{ + tree::AggregateData, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, + }; #[test] fn encode_leaf_tree() { diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index ccad4dd2a..5047fe594 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -12,7 +12,10 @@ use integer_encoding::VarInt; #[cfg(feature = "full")] use super::hash::{CryptoHash, HASH_LENGTH, NULL_HASH}; -use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; +use crate::{ + merk::NodeType, + tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, +}; #[cfg(feature = "full")] use crate::{ tree::{ @@ -21,7 +24,6 @@ use crate::{ }, Link, HASH_LENGTH_U32, HASH_LENGTH_U32_X2, }; -use crate::merk::NodeType; // TODO: maybe use something similar to Vec but without capacity field, // (should save 16 bytes per entry). also, maybe a shorter length // field to save even more. also might be possible to combine key @@ -418,11 +420,7 @@ impl KV { let key_len = self.key.len() as u32; let node_type = self.feature_type.node_type(); - Self::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_cost, - node_type, - ) + Self::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_cost, node_type) } /// This function is used to calculate the cost of groveDB sum item nodes diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index 2537b15bd..4b8c9f650 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -2,6 +2,7 @@ #[cfg(feature = "full")] use std::io::{Read, Write}; + use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "full")] use ed::{Decode, Encode, Result, Terminated}; @@ -10,11 +11,11 @@ use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(feature = "full")] use super::{hash::CryptoHash, TreeNode}; -#[cfg(feature = "full")] -use crate::HASH_LENGTH_U32; use crate::merk::NodeType; #[cfg(feature = "full")] use crate::tree::tree_feature_type::AggregateData; +#[cfg(feature = "full")] +use crate::HASH_LENGTH_U32; // TODO: optimize memory footprint #[cfg(feature = "full")] @@ -271,7 +272,11 @@ impl Link { debug_assert!(self.key().len() < 256, "Key length must be less than 256"); Ok(match self { - Link::Reference { key, aggregate_data, .. } => match aggregate_data { + Link::Reference { + key, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => key.len() + 36, // 1 + HASH_LENGTH + 2 + 1, AggregateData::Count(_) | AggregateData::Sum(_) => { // 1 for key len @@ -297,7 +302,16 @@ impl Link { } }, Link::Modified { .. } => panic!("No encoding for Link::Modified"), - Link::Uncommitted { tree, aggregate_data, .. } | Link::Loaded { tree, aggregate_data, .. } => match aggregate_data { + Link::Uncommitted { + tree, + aggregate_data, + .. + } + | Link::Loaded { + tree, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => tree.key().len() + 36, // 1 + 32 + 2 + 1, AggregateData::Count(_) | AggregateData::Sum(_) => { tree.key().len() + 44 // 1 + 32 + 2 + 1 + 8 @@ -359,7 +373,7 @@ impl Encode for Link { out.write_i128::(*big_sum_value)?; } AggregateData::Count(count_value) => { - out.write_all(&[2])?; + out.write_all(&[3])?; out.write_varint(*count_value)?; } } @@ -372,7 +386,11 @@ impl Encode for Link { debug_assert!(self.key().len() < 256, "Key length must be less than 256"); Ok(match self { - Link::Reference { key, aggregate_data, .. } => match aggregate_data { + Link::Reference { + key, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => key.len() + 36, // 1 + 32 + 2 + 1 AggregateData::Sum(sum_value) => { let encoded_sum_value = sum_value.encode_var_vec(); @@ -411,7 +429,16 @@ impl Encode for Link { } }, Link::Modified { .. } => panic!("No encoding for Link::Modified"), - Link::Uncommitted { tree, aggregate_data, .. } | Link::Loaded { tree, aggregate_data, .. } => match aggregate_data { + Link::Uncommitted { + tree, + aggregate_data, + .. + } + | Link::Loaded { + tree, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => tree.key().len() + 36, // 1 + 32 + 2 + 1 AggregateData::Sum(sum_value) => { let encoded_sum_value = sum_value.encode_var_vec(); @@ -708,7 +735,7 @@ mod test { bytes, vec![ 3, 1, 2, 3, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 1, 100, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 3, 50, ] ); } @@ -721,7 +748,7 @@ mod test { child_heights: (123, 124), hash: [55; 32], }; - assert_eq!(link.encoding_length().unwrap(), 40); + assert_eq!(link.encoding_length().unwrap(), 55); let mut bytes = vec![]; link.encode_into(&mut bytes).unwrap(); @@ -731,7 +758,8 @@ mod test { bytes, vec![ 3, 1, 2, 3, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 1, 100, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 2, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50 ] ); } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index ed9d470e2..2d24825eb 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -60,12 +60,13 @@ pub use link::Link; #[cfg(feature = "full")] pub use ops::{AuxMerkBatch, BatchEntry, MerkBatch, Op, PanicSource}; #[cfg(any(feature = "full", feature = "verify"))] -pub use tree_feature_type::TreeFeatureType; -#[cfg(any(feature = "full", feature = "verify"))] pub use tree_feature_type::AggregateData; +#[cfg(any(feature = "full", feature = "verify"))] +pub use tree_feature_type::TreeFeatureType; #[cfg(feature = "full")] pub use walk::{Fetch, RefWalker, Walker}; +use crate::merk::NodeType; #[cfg(feature = "full")] use crate::tree::hash::HASH_LENGTH_X2; #[cfg(feature = "full")] @@ -74,7 +75,6 @@ use crate::tree::kv::ValueDefinedCostType; use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "full")] use crate::{error::Error, Error::Overflow}; -use crate::merk::NodeType; // TODO: remove need for `TreeInner`, and just use `Box` receiver for // relevant methods @@ -99,7 +99,6 @@ impl TreeNodeInner { (self.kv.value, self.kv.feature_type) } - /// Get the value as slice of the key value struct pub fn value_as_slice(&self) -> &[u8] { self.kv.value.as_slice() @@ -461,7 +460,7 @@ impl TreeNode { AggregateData::Sum(s) => s.encode_var_vec().len() as u32, AggregateData::BigSum(s) => 16 as u32, AggregateData::Count(c) => c.encode_var_vec().len() as u32, - } + }, ) }) } @@ -507,7 +506,9 @@ impl TreeNode { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => Ok(0), AggregateData::Sum(s) => Ok(s), - AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree("for aggregate data as i64".to_string())), + AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree( + "for aggregate data as i64".to_string(), + )), AggregateData::Count(c) => { if c > i64::MAX as u64 { Err(Overflow("count overflow when below sum tree")) @@ -515,7 +516,7 @@ impl TreeNode { Ok(c as i64) } } - } + }, _ => Ok(0), } } @@ -533,10 +534,12 @@ impl TreeNode { } else { Ok(s as u64) } - }, - AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree("for aggregate data as u64".to_string())), - AggregateData::Count(c) => Ok(c) - } + } + AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree( + "for aggregate data as u64".to_string(), + )), + AggregateData::Count(c) => Ok(c), + }, _ => Ok(0), } } @@ -551,7 +554,7 @@ impl TreeNode { AggregateData::Sum(s) => s as i128, AggregateData::BigSum(s) => s, AggregateData::Count(c) => c as i128, - } + }, _ => 0, } } @@ -580,7 +583,7 @@ impl TreeNode { .and_then(|a| a.checked_add(right)) .ok_or(Overflow("sum is overflowing")) .map(AggregateData::Sum) - }, + } TreeFeatureType::BigSummedMerkNode(value) => value .checked_add(self.child_aggregate_data_as_i128(true)) .and_then(|a| a.checked_add(self.child_aggregate_data_as_i128(false))) @@ -594,7 +597,7 @@ impl TreeNode { .and_then(|a| a.checked_add(right)) .ok_or(Overflow("count is overflowing")) .map(AggregateData::Count) - }, + } } } @@ -1118,7 +1121,7 @@ pub const fn side_to_str(left: bool) -> &'static str { #[cfg(test)] mod test { - use super::{commit::NoopCommit, hash::NULL_HASH, TreeNode}; + use super::{commit::NoopCommit, hash::NULL_HASH, AggregateData, TreeNode}; use crate::tree::{ tree_feature_type::TreeFeatureType::SummedMerkNode, TreeFeatureType::BasicMerkNode, }; @@ -1327,6 +1330,10 @@ mod test { .unwrap() .expect("commit failed"); - assert_eq!(Some(8), tree.sum().expect("expected to get sum from tree")); + assert_eq!( + AggregateData::Sum(8), + tree.aggregate_data() + .expect("expected to get sum from tree") + ); } } diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 3e10b2c81..690e136fe 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -1100,7 +1100,7 @@ mod test { None, Some(Link::Loaded { hash: [123; 32], - sum: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (0, 0), tree: TreeNode::new(b"foo2".to_vec(), b"bar2".to_vec(), None, BasicMerkNode) .unwrap(), diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 7dd6472e2..1ecf967e8 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -2,6 +2,7 @@ #[cfg(any(feature = "full", feature = "verify"))] use std::io::{Read, Write}; + use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "full")] use ed::Terminated; @@ -9,10 +10,13 @@ use ed::Terminated; use ed::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; -use crate::merk::NodeType; + #[cfg(any(feature = "full", feature = "verify"))] use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; -use crate::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; +use crate::{ + merk::{NodeType, TreeType}, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, +}; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -49,28 +53,37 @@ pub enum AggregateData { } impl AggregateData { + pub fn parent_tree_type(&self) -> TreeType { + match self { + AggregateData::NoAggregateData => TreeType::NormalTree, + AggregateData::Sum(_) => TreeType::SumTree, + AggregateData::BigSum(_) => TreeType::BigSumTree, + AggregateData::Count(_) => TreeType::CountTree, + } + } + pub fn as_i64(&self) -> i64 { - match self { - AggregateData::NoAggregateData => 0, - AggregateData::Sum(s) => *s, - AggregateData::BigSum(i) => { - let max = i64::MAX as i128; - if *i > max { - i64::MAX - } else { - *i as i64 + match self { + AggregateData::NoAggregateData => 0, + AggregateData::Sum(s) => *s, + AggregateData::BigSum(i) => { + let max = i64::MAX as i128; + if *i > max { + i64::MAX + } else { + *i as i64 + } } - } - AggregateData::Count(c) => { - let max = i64::MAX as u64; - if *c > max { - i64::MAX - } else { - *c as i64 + AggregateData::Count(c) => { + let max = i64::MAX as u64; + if *c > max { + i64::MAX + } else { + *c as i64 + } } } } -} pub fn as_u64(&self) -> u64 { match self { @@ -81,7 +94,7 @@ impl AggregateData { } else { *s as u64 } - }, + } AggregateData::BigSum(i) => { let max = u64::MAX as i128; if *i > max { @@ -92,24 +105,16 @@ impl AggregateData { *i as u64 } } - AggregateData::Count(c) => { - *c - } + AggregateData::Count(c) => *c, } } pub fn as_i128(&self) -> i128 { match self { AggregateData::NoAggregateData => 0, - AggregateData::Sum(s) => { - *s as i128 - }, - AggregateData::BigSum(i) => { - *i - } - AggregateData::Count(c) => { - *c as i128 - } + AggregateData::Sum(s) => *s as i128, + AggregateData::BigSum(i) => *i, + AggregateData::Count(c) => *c as i128, } } } @@ -189,9 +194,7 @@ impl Encode for TreeFeatureType { // encoded_sum.len() for the length of the encoded vector Ok(1 + encoded_sum.len()) } - BigSummedMerkNode(_) => { - Ok(17) - } + BigSummedMerkNode(_) => Ok(17), CountedMerkNode(count) => { let encoded_sum = count.encode_var_vec(); // 1 for the enum type diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 2274f804f..f0a0aeff4 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -403,7 +403,7 @@ mod test { use grovedb_version::version::GroveVersion; use super::{super::NoopCommit, *}; - use crate::tree::{TreeFeatureType::BasicMerkNode, TreeNode}; + use crate::tree::{AggregateData, TreeFeatureType::BasicMerkNode, TreeNode}; #[derive(Clone)] struct MockSource {} @@ -491,7 +491,7 @@ mod test { hash: Default::default(), key: b"foo".to_vec(), child_heights: (0, 0), - aggregate_data: None, + aggregate_data: AggregateData::NoAggregateData, }), None, BasicMerkNode, diff --git a/node-grove/src/converter.rs b/node-grove/src/converter.rs index a822faadc..2aca7b0d9 100644 --- a/node-grove/src/converter.rs +++ b/node-grove/src/converter.rs @@ -38,6 +38,8 @@ fn element_to_string(element: Element) -> String { Element::Reference(..) => "reference".to_string(), Element::Tree(..) => "tree".to_string(), Element::SumTree(..) => "sum_tree".to_string(), + Element::BigSumTree(..) => "big_sum_tree".to_string(), + Element::CountTree(..) => "big_sum_tree".to_string(), } } @@ -92,6 +94,8 @@ pub fn element_to_js_object<'a, C: Context<'a>>( Element::Reference(..) => nested_vecs_to_js(vec![], cx)?, Element::Tree(..) => nested_vecs_to_js(vec![], cx)?, Element::SumTree(..) => nested_vecs_to_js(vec![], cx)?, + Element::BigSumTree(..) => nested_vecs_to_js(vec![], cx)?, + Element::CountTree(..) => nested_vecs_to_js(vec![], cx)?, }; js_object.set(cx, "value", js_value)?; From a46c4dae0ebee82955ef5b6b43bb3feece20a570 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 8 Jan 2025 12:52:37 +0700 Subject: [PATCH 06/30] fixes and tests --- grovedb-version/src/version/v2.rs | 4 +- grovedb/Cargo.toml | 1 + grovedb/src/batch/mod.rs | 22 + grovedb/src/element/constructor.rs | 12 + grovedb/src/element/helpers.rs | 27 +- grovedb/src/lib.rs | 39 +- grovedb/src/operations/get/mod.rs | 5 +- grovedb/src/operations/insert/mod.rs | 91 +- grovedb/src/operations/proof/generate.rs | 5 +- grovedb/src/tests/count_tree_tests.rs | 852 +++++++ grovedb/src/tests/mod.rs | 1 + grovedb/src/tests/sum_tree_tests.rs | 1999 +++++++++-------- .../src/estimated_costs/average_case_costs.rs | 5 +- merk/src/tree/mod.rs | 6 +- 14 files changed, 2174 insertions(+), 895 deletions(-) create mode 100644 grovedb/src/tests/count_tree_tests.rs diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 304b034c1..6bb0e31ee 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -13,7 +13,7 @@ use crate::version::{ }; pub const GROVE_V2: GroveVersion = GroveVersion { - protocol_version: 0, + protocol_version: 1, grovedb_versions: GroveDBVersions { apply_batch: GroveDBApplyBatchVersions { apply_batch_structure: 0, @@ -186,7 +186,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { }, merk_versions: MerkVersions { average_case_costs: MerkAverageCaseCostsVersions { - add_average_case_merk_propagate: 1, + add_average_case_merk_propagate: 1, // changed }, }, }; diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index c294463dc..02fca1458 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -45,6 +45,7 @@ criterion = "0.5.1" hex = "0.4.3" pretty_assertions = "1.4.0" rand = "0.8.5" +assert_matches = "1.5.0" [[bench]] name = "insertion_benchmark" diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 97b0ed721..c14426ef7 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1864,6 +1864,28 @@ impl GroveDb { aggregate_data, } .into(); + } else if let Element::BigSumTree(.., flags) = + element + { + *mutable_occupied_entry = + GroveOp::InsertTreeWithRootHash { + hash: root_hash, + root_key: calculated_root_key, + flags: flags.clone(), + aggregate_data, + } + .into(); + } else if let Element::CountTree(.., flags) = + element + { + *mutable_occupied_entry = + GroveOp::InsertTreeWithRootHash { + hash: root_hash, + root_key: calculated_root_key, + flags: flags.clone(), + aggregate_data, + } + .into(); } else { return Err(Error::InvalidBatchOperation( "insertion of element under a non tree", diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 556df1dac..0ba0fd8ff 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -29,6 +29,18 @@ impl Element { Element::new_sum_tree(Default::default()) } + #[cfg(feature = "full")] + /// Set element to default empty big sum tree without flags + pub fn empty_big_sum_tree() -> Self { + Element::new_big_sum_tree(Default::default()) + } + + #[cfg(feature = "full")] + /// Set element to default empty count tree without flags + pub fn empty_count_tree() -> Self { + Element::new_count_tree(Default::default()) + } + #[cfg(feature = "full")] /// Set element to default empty sum tree with flags pub fn empty_sum_tree_with_flags(flags: Option) -> Self { diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index f22a2c7e4..8b8a22a18 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -21,7 +21,7 @@ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::Grove #[cfg(feature = "full")] use integer_encoding::VarInt; -use crate::element::BIG_SUM_TREE_COST_SIZE; +use crate::element::{BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE}; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; #[cfg(any(feature = "full", feature = "verify"))] @@ -45,6 +45,16 @@ impl Element { } } + #[cfg(any(feature = "full", feature = "verify"))] + /// Decoded the integer value in the CountTree element type, returns 1 for + /// everything else + pub fn count_value_or_default(&self) -> u64 { + match self { + Element::CountTree(_, count_value, _) => *count_value, + _ => 1, + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Decoded the integer value in the SumItem element type, returns 0 for /// everything else @@ -232,7 +242,7 @@ impl Element { TreeType::NormalTree => Ok(BasicMerkNode), TreeType::SumTree => Ok(SummedMerkNode(self.sum_value_or_default())), TreeType::BigSumTree => Ok(BigSummedMerkNode(self.big_sum_value_or_default())), - TreeType::CountTree => Ok(CountedMerkNode(1)), + TreeType::CountTree => Ok(CountedMerkNode(self.count_value_or_default())), } } @@ -388,6 +398,17 @@ impl Element { key_len, value_len, node_type, ) } + Element::CountTree(_, _count_value, flags) => { + let flags_len = flags.map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let value_len = COUNT_TREE_COST_SIZE + flags_len; + let key_len = key.len() as u32; + KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key_len, value_len, node_type, + ) + } Element::SumItem(.., flags) => { let flags_len = flags.map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -414,6 +435,7 @@ impl Element { Element::SumTree(..) => Ok(SUM_TREE_COST_SIZE), Element::BigSumTree(..) => Ok(BIG_SUM_TREE_COST_SIZE), Element::SumItem(..) => Ok(SUM_ITEM_COST_SIZE), + Element::CountTree(..) => Ok(COUNT_TREE_COST_SIZE), _ => Err(Error::CorruptedCodeExecution( "trying to get tree cost from non tree element", )), @@ -436,6 +458,7 @@ impl Element { Element::Tree(..) => Some(LayeredValueDefinedCost(cost)), Element::SumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::BigSumTree(..) => Some(LayeredValueDefinedCost(cost)), + Element::CountTree(..) => Some(LayeredValueDefinedCost(cost)), Element::SumItem(..) => Some(SpecializedValueDefinedCost(cost)), _ => None, } diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 84cfeac53..36907b094 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -501,7 +501,8 @@ impl GroveDb { let tree_type = element.tree_type(); if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) - | Element::BigSumTree(root_key, ..) = element + | Element::BigSumTree(root_key, ..) + | Element::CountTree(root_key, ..) = element { let tree_type = tree_type.expect("expected tree type"); Merk::open_layered_with_root_key( @@ -715,7 +716,7 @@ impl GroveDb { grove_version ) ); - let (root_hash, root_key, sum) = cost_return_on_error!( + let (root_hash, root_key, aggregate_data) = cost_return_on_error!( &mut cost, child_tree .root_hash_key_and_aggregate_data() @@ -728,7 +729,7 @@ impl GroveDb { parent_key, root_key, root_hash, - sum, + aggregate_data, grove_version, ) ); @@ -832,6 +833,19 @@ impl GroveDb { None, grove_version, ) + } else if let Element::CountTree(.., flag) = element { + let tree = Element::new_count_tree_with_flags_and_count_value( + maybe_root_key, + aggregate_data.as_u64(), + flag, + ); + tree.insert_subtree( + parent_tree, + key.as_ref(), + root_tree_hash, + None, + grove_version, + ) } else { Err(Error::InvalidPath( "can only propagate on tree items".to_owned(), @@ -912,6 +926,25 @@ impl GroveDb { merk_feature_type, grove_version, ) + } else if let Element::CountTree(.., flag) = element { + let tree = Element::new_count_tree_with_flags_and_count_value( + maybe_root_key, + aggregate_data.as_u64(), + flag, + ); + let merk_feature_type = cost_return_on_error!( + &mut cost, + tree.get_feature_type(parent_tree.tree_type) + .wrap_with_cost(OperationCost::default()) + ); + tree.insert_subtree_into_batch_operations( + key, + root_tree_hash, + true, + batch_operations, + merk_feature_type, + grove_version, + ) } else { Err(Error::InvalidPath( "can only propagate on tree items".to_owned(), diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index b62896996..073e02abe 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -455,7 +455,10 @@ impl GroveDb { } .unwrap_add_cost(&mut cost); match element { - Ok(Element::Tree(..)) | Ok(Element::SumTree(..)) => Ok(()).wrap_with_cost(cost), + Ok(Element::Tree(..)) + | Ok(Element::SumTree(..)) + | Ok(Element::BigSumTree(..)) + | Ok(Element::CountTree(..)) => Ok(()).wrap_with_cost(cost), Ok(_) | Err(Error::PathKeyNotFound(_)) => Err(error_fn()).wrap_with_cost(cost), Err(e) => Err(e).wrap_with_cost(cost), } diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 5926fedd2..fc357cd82 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -317,7 +317,10 @@ impl GroveDb { ) ); } - Element::Tree(ref value, _) | Element::SumTree(ref value, ..) => { + Element::Tree(ref value, _) + | Element::SumTree(ref value, ..) + | Element::BigSumTree(ref value, ..) + | Element::CountTree(ref value, ..) => { if value.is_some() { return Err(Error::InvalidCodeExecution( "a tree should be empty at the moment of insertion when not using batches", @@ -450,7 +453,10 @@ impl GroveDb { ) ); } - Element::Tree(ref value, _) | Element::SumTree(ref value, ..) => { + Element::Tree(ref value, _) + | Element::SumTree(ref value, ..) + | Element::BigSumTree(ref value, ..) + | Element::CountTree(ref value, ..) => { if value.is_some() { return Err(Error::InvalidCodeExecution( "a tree should be empty at the moment of insertion when not using batches", @@ -1593,6 +1599,87 @@ mod tests { ); } + #[test] + fn test_one_insert_item_cost_under_count_tree() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_count_tree(), + None, + Some(&tx), + grove_version, + ) + .unwrap() + .unwrap(); + + let cost = db + .insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item(b"test".to_vec()), + None, + Some(&tx), + grove_version, + ) + .cost_as_result() + .unwrap(); + + // Explanation for 152 storage_written_bytes + + // Key -> 37 bytes + // 32 bytes for the key prefix + // 4 bytes for the key + // 1 byte for key_size (required space for 36) + + // Value -> 81 + // 1 for the enum type item + // 1 for size of test bytes + // 4 for test bytes + // 1 for the flag option (but no flags) + // 32 for node hash + // 32 for value hash (trees have this for free) + // 9 for Count node + // 1 byte for the value_size (required space for 1) + + // Parent Hook -> 48 + // Key Bytes 4 + // Hash Size 32 + // Key Length 1 + // Count Merk 9 + // Child Heights 2 + + // Total 37 + 81 + 48 = 166 + + // Explanation for replaced bytes + + // Replaced parent Value -> 86 + // 1 for the flag option (but no flags) + // 1 for the enum type + // 1 for an empty option + // 1 for the count merk + // 9 for the count + // 32 for node hash + // 40 for the parent hook + // 2 byte for the value_size + assert_eq!( + cost, + OperationCost { + seek_count: 5, // todo: verify this + storage_cost: StorageCost { + added_bytes: 166, + replaced_bytes: 87, + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 162, // todo: verify this + hash_node_calls: 8, // todo: verify this + } + ); + } + #[test] fn test_one_insert_item_with_apple_flags_cost() { let grove_version = GroveVersion::latest(); diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 6e814f67f..7708d5698 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -314,7 +314,10 @@ impl GroveDb { } has_a_result_at_level |= true; } - Ok(Element::Tree(Some(_), _)) | Ok(Element::SumTree(Some(_), ..)) + Ok(Element::Tree(Some(_), _)) + | Ok(Element::SumTree(Some(_), ..)) + | Ok(Element::BigSumTree(Some(_), ..)) + | Ok(Element::CountTree(Some(_), ..)) if !done_with_results && query.has_subquery_or_matching_in_path_on_key(key) => { diff --git a/grovedb/src/tests/count_tree_tests.rs b/grovedb/src/tests/count_tree_tests.rs new file mode 100644 index 000000000..6ac71ed02 --- /dev/null +++ b/grovedb/src/tests/count_tree_tests.rs @@ -0,0 +1,852 @@ +//! Count tree tests + +#[cfg(test)] +mod tests { + use assert_matches::assert_matches; + use grovedb_merk::{ + proofs::Query, + tree::{kv::ValueDefinedCostType, AggregateData}, + TreeFeatureType::{BasicMerkNode, CountedMerkNode}, + }; + use grovedb_storage::StorageBatch; + use grovedb_version::version::GroveVersion; + + use crate::{ + batch::QualifiedGroveDbOp, + reference_path::ReferencePathType, + tests::{make_test_grovedb, TEST_LEAF}, + Element, Error, GroveDb, PathQuery, + }; + + #[test] + fn test_count_tree_behaves_like_regular_tree() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + + // Can fetch count tree + let count_tree = db + .get([TEST_LEAF].as_ref(), b"key", None, grove_version) + .unwrap() + .expect("should get count tree"); + assert!(matches!(count_tree, Element::CountTree(..))); + + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"innerkey", + Element::new_item(vec![1]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"innerkey2", + Element::new_item(vec![3]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"innerkey3", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + // Test proper item retrieval + let item = db + .get( + [TEST_LEAF, b"key"].as_ref(), + b"innerkey", + None, + grove_version, + ) + .unwrap() + .expect("should get item"); + assert_eq!(item, Element::new_item(vec![1])); + + // Test proof generation + let mut query = Query::new(); + query.insert_key(b"innerkey2".to_vec()); + + let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"key".to_vec()], query); + let proof = db + .prove_query(&path_query, None, grove_version) + .unwrap() + .expect("should generate proof"); + let (root_hash, result_set) = GroveDb::verify_query_raw(&proof, &path_query, grove_version) + .expect("should verify proof"); + assert_eq!( + root_hash, + db.grove_db.root_hash(None, grove_version).unwrap().unwrap() + ); + assert_eq!(result_set.len(), 1); + assert_eq!( + Element::deserialize(&result_set[0].value, grove_version) + .expect("should deserialize element"), + Element::new_item(vec![3]) + ); + } + + #[test] + fn test_homogenous_node_type_in_count_trees_and_regular_trees() { + let grove_version = GroveVersion::latest(); + // All elements in a count tree must have a count feature type + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + // Add count items + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item1", + Element::new_item(vec![30]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item2", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + // Add regular items + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item3", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item4", + Element::new_item(vec![15]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + let batch = StorageBatch::new(); + + // Open merk and check all elements in it + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + let feature_type_node_1 = merk + .get_feature_type( + b"item1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + let feature_type_node_2 = merk + .get_feature_type( + b"item2", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + let feature_type_node_3 = merk + .get_feature_type( + b"item3", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + let feature_type_node_4 = merk + .get_feature_type( + b"item4", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + + assert_matches!(feature_type_node_1, CountedMerkNode(1)); + assert_matches!(feature_type_node_2, CountedMerkNode(1)); + assert_matches!(feature_type_node_3, CountedMerkNode(1)); + assert_matches!(feature_type_node_4, CountedMerkNode(1)); + + // Perform the same test on regular trees + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item1", + Element::new_item(vec![30]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item2", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + merk.get_feature_type( + b"item1", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(BasicMerkNode) + )); + assert!(matches!( + merk.get_feature_type( + b"item2", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(BasicMerkNode) + )); + assert_eq!( + merk.aggregate_data().expect("expected to get count"), + AggregateData::NoAggregateData + ); + } + + #[test] + fn test_count_tree_feature() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + + let batch = StorageBatch::new(); + + // Sum should be non for non count tree + // TODO: change interface to retrieve element directly + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get count"), + AggregateData::NoAggregateData + ); + + // Add count tree + db.insert( + [TEST_LEAF].as_ref(), + b"key2", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert count tree"); + let count_tree = db + .get([TEST_LEAF].as_ref(), b"key2", None, grove_version) + .unwrap() + .expect("should retrieve tree"); + assert_eq!(count_tree.count_value_or_default(), 0); + + // Add count items to the count tree + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item1", + Element::new_item(vec![30]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + // TODO: change interface to retrieve element directly + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get count"), + AggregateData::Count(1) + ); + + // Add more count items + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item2", + Element::new_item(vec![3]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item3", + Element::new_item(vec![3]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get count"), + AggregateData::Count(3) + ); + + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item4", + Element::new_item(vec![29]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get count"), + AggregateData::Count(4) + ); + + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item2", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item3", + Element::new_item(vec![3]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get count"), + AggregateData::Count(4) + ); + + db.delete( + [TEST_LEAF, b"key2"].as_ref(), + b"item4", + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to delete"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get count"), + AggregateData::Count(3) + ); + } + + #[test] + fn test_count_tree_propagation() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + // Tree + // count_key: CountTree + // / \ + // countitem3 tree2: CountTree + // + // tree2 : CountTree + // / + // item1 item2 item3 ref1 + db.insert( + [TEST_LEAF].as_ref(), + b"count_key", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"count_key"].as_ref(), + b"tree2", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"count_key"].as_ref(), + b"countitem3", + Element::new_item(vec![3]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"count_key", b"tree2"].as_ref(), + b"item1", + Element::new_item(vec![2]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"count_key", b"tree2"].as_ref(), + b"item2", + Element::new_item(vec![5]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"count_key", b"tree2"].as_ref(), + b"item3", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"count_key", b"tree2"].as_ref(), + b"ref1", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"count_key".to_vec(), + b"tree2".to_vec(), + b"item1".to_vec(), + ])), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + let count_tree = db + .get([TEST_LEAF].as_ref(), b"count_key", None, grove_version) + .unwrap() + .expect("should fetch tree"); + assert_eq!(count_tree.count_value_or_default(), 5); + + let batch = StorageBatch::new(); + + // Assert node feature types + let test_leaf_merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + let root_tree_feature_type = test_leaf_merk + .get_feature_type( + b"count_key", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("tree feature type"); + + assert_matches!(root_tree_feature_type, BasicMerkNode); + + let parent_count_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"count_key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + let count_tree_feature_type = parent_count_tree + .get_feature_type( + b"tree2", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("tree feature type"); + assert_matches!(count_tree_feature_type, CountedMerkNode(4)); + + let child_count_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"count_key", b"tree2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + let count_tree_feature_type = child_count_tree + .get_feature_type( + b"item1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("tree feature type"); + + assert_matches!(count_tree_feature_type, CountedMerkNode(1)); + + let count_tree_feature_type = child_count_tree + .get_feature_type( + b"item2", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("tree feature type"); + + assert_matches!(count_tree_feature_type, CountedMerkNode(1)); + + let count_tree_feature_type = child_count_tree + .get_feature_type( + b"item3", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("tree feature type"); + + assert_matches!(count_tree_feature_type, CountedMerkNode(1)); + + let count_tree_feature_type = child_count_tree + .get_feature_type( + b"ref1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("tree feature type"); + + assert_matches!(count_tree_feature_type, CountedMerkNode(1)); + } + + #[test] + fn test_count_tree_with_batches() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"key1".to_vec(), + Element::empty_count_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"a".to_vec(), + Element::new_item(vec![214]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"b".to_vec(), + Element::new_item(vec![10]), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let count_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + + let tree_feature_type_a = count_tree + .get_feature_type( + b"a", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected tree feature type"); + + let tree_feature_type_b = count_tree + .get_feature_type( + b"a", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected tree feature type"); + + assert_matches!(tree_feature_type_a, CountedMerkNode(1)); + assert_matches!(tree_feature_type_b, CountedMerkNode(1)); + + // Create new batch to use existing tree + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"c".to_vec(), + Element::new_item(vec![10]), + )]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let count_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + let tree_feature_type_c = count_tree + .get_feature_type( + b"c", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected tree feature type"); + assert_matches!(tree_feature_type_c, CountedMerkNode(1)); + assert_eq!( + count_tree.aggregate_data().expect("expected to get count"), + AggregateData::Count(3) + ); + + // Test propagation + // Add a new count tree with its own count items, should affect count of + // original tree + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"d".to_vec(), + Element::empty_count_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], + b"first".to_vec(), + Element::new_item(vec![2]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], + b"second".to_vec(), + Element::new_item(vec![4]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"e".to_vec(), + Element::empty_count_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"first".to_vec(), + Element::new_item(vec![3]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"second".to_vec(), + Element::new_item(vec![4]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"third".to_vec(), + Element::empty_count_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![ + TEST_LEAF.to_vec(), + b"key1".to_vec(), + b"e".to_vec(), + b"third".to_vec(), + ], + b"a".to_vec(), + Element::new_item(vec![5]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![ + TEST_LEAF.to_vec(), + b"key1".to_vec(), + b"e".to_vec(), + b"third".to_vec(), + ], + b"b".to_vec(), + Element::new_item(vec![5]), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let count_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + count_tree.aggregate_data().expect("expected to get count"), + AggregateData::Count(9) + ); + } +} diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 896346c56..bd6561ff1 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -6,6 +6,7 @@ mod query_tests; mod sum_tree_tests; +mod count_tree_tests; mod tree_hashes_tests; use std::{ diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 1371e27e5..215249ca1 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -1,754 +1,285 @@ //! Sum tree tests -use grovedb_merk::{ - proofs::Query, - tree::{kv::ValueDefinedCostType, AggregateData}, - TreeFeatureType::{BasicMerkNode, SummedMerkNode}, -}; -use grovedb_storage::StorageBatch; -use grovedb_version::version::GroveVersion; - -use crate::{ - batch::QualifiedGroveDbOp, - reference_path::ReferencePathType, - tests::{make_test_grovedb, TEST_LEAF}, - Element, Error, GroveDb, PathQuery, -}; - -#[test] -fn test_sum_tree_behaves_like_regular_tree() { - let grove_version = GroveVersion::latest(); - let db = make_test_grovedb(grove_version); - db.insert( - [TEST_LEAF].as_ref(), - b"key", - Element::empty_sum_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - - // Can fetch sum tree - let sum_tree = db - .get([TEST_LEAF].as_ref(), b"key", None, grove_version) - .unwrap() - .expect("should get tree"); - assert!(matches!(sum_tree, Element::SumTree(..))); - - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"innerkey", - Element::new_item(vec![1]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"innerkey2", - Element::new_item(vec![3]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"innerkey3", - Element::empty_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - - // Test proper item retrieval - let item = db - .get( +#[cfg(test)] +mod tests { + use grovedb_merk::{ + proofs::Query, + tree::{kv::ValueDefinedCostType, AggregateData}, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, + }; + use grovedb_storage::StorageBatch; + use grovedb_version::version::GroveVersion; + + use crate::{ + batch::QualifiedGroveDbOp, + element::SumValue, + reference_path::ReferencePathType, + tests::{make_test_grovedb, TEST_LEAF}, + Element, Error, GroveDb, PathQuery, + }; + + #[test] + fn test_sum_tree_behaves_like_regular_tree() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + + // Can fetch sum tree + let sum_tree = db + .get([TEST_LEAF].as_ref(), b"key", None, grove_version) + .unwrap() + .expect("should get tree"); + assert!(matches!(sum_tree, Element::SumTree(..))); + + db.insert( [TEST_LEAF, b"key"].as_ref(), b"innerkey", + Element::new_item(vec![1]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"innerkey2", + Element::new_item(vec![3]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"innerkey3", + Element::empty_tree(), + None, None, grove_version, ) .unwrap() - .expect("should get item"); - assert_eq!(item, Element::new_item(vec![1])); + .expect("should insert item"); - // Test proof generation - let mut query = Query::new(); - query.insert_key(b"innerkey2".to_vec()); + // Test proper item retrieval + let item = db + .get( + [TEST_LEAF, b"key"].as_ref(), + b"innerkey", + None, + grove_version, + ) + .unwrap() + .expect("should get item"); + assert_eq!(item, Element::new_item(vec![1])); - let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"key".to_vec()], query); - let proof = db - .prove_query(&path_query, None, grove_version) - .unwrap() - .expect("should generate proof"); - let (root_hash, result_set) = - GroveDb::verify_query_raw(&proof, &path_query, grove_version).expect("should verify proof"); - assert_eq!( - root_hash, - db.grove_db.root_hash(None, grove_version).unwrap().unwrap() - ); - assert_eq!(result_set.len(), 1); - assert_eq!( - Element::deserialize(&result_set[0].value, grove_version) - .expect("should deserialize element"), - Element::new_item(vec![3]) - ); -} + // Test proof generation + let mut query = Query::new(); + query.insert_key(b"innerkey2".to_vec()); -#[test] -fn test_sum_item_behaves_like_regular_item() { - let grove_version = GroveVersion::latest(); - let db = make_test_grovedb(grove_version); - db.insert( - [TEST_LEAF].as_ref(), - b"sumkey", - Element::empty_sum_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - db.insert( - [TEST_LEAF, b"sumkey"].as_ref(), - b"k1", - Element::new_item(vec![1]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - db.insert( - [TEST_LEAF, b"sumkey"].as_ref(), - b"k2", - Element::new_sum_item(5), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - db.insert( - [TEST_LEAF, b"sumkey"].as_ref(), - b"k3", - Element::empty_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - - // Test proper item retrieval - let item = db - .get([TEST_LEAF, b"sumkey"].as_ref(), b"k2", None, grove_version) - .unwrap() - .expect("should get item"); - assert_eq!(item, Element::new_sum_item(5)); - - // Test proof generation - let mut query = Query::new(); - query.insert_key(b"k2".to_vec()); - - let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"sumkey".to_vec()], query); - let proof = db - .prove_query(&path_query, None, grove_version) - .unwrap() - .expect("should generate proof"); - let (root_hash, result_set) = - GroveDb::verify_query_raw(&proof, &path_query, grove_version).expect("should verify proof"); - assert_eq!( - root_hash, - db.grove_db.root_hash(None, grove_version).unwrap().unwrap() - ); - assert_eq!(result_set.len(), 1); - let element_from_proof = Element::deserialize(&result_set[0].value, grove_version) - .expect("should deserialize element"); - assert_eq!(element_from_proof, Element::new_sum_item(5)); - assert_eq!(element_from_proof.sum_value_or_default(), 5); -} + let path_query = PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"key".to_vec()], query); + let proof = db + .prove_query(&path_query, None, grove_version) + .unwrap() + .expect("should generate proof"); + let (root_hash, result_set) = GroveDb::verify_query_raw(&proof, &path_query, grove_version) + .expect("should verify proof"); + assert_eq!( + root_hash, + db.grove_db.root_hash(None, grove_version).unwrap().unwrap() + ); + assert_eq!(result_set.len(), 1); + assert_eq!( + Element::deserialize(&result_set[0].value, grove_version) + .expect("should deserialize element"), + Element::new_item(vec![3]) + ); + } -#[test] -fn test_cannot_insert_sum_item_in_regular_tree() { - let grove_version = GroveVersion::latest(); - let db = make_test_grovedb(grove_version); - db.insert( - [TEST_LEAF].as_ref(), - b"sumkey", - Element::empty_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - assert!(matches!( + #[test] + fn test_sum_item_behaves_like_regular_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); db.insert( - [TEST_LEAF, b"sumkey"].as_ref(), - b"k1", - Element::new_sum_item(5), + [TEST_LEAF].as_ref(), + b"sumkey", + Element::empty_sum_tree(), None, None, - grove_version - ) - .unwrap(), - Err(Error::InvalidInput("cannot add sum item to non sum tree")) - )); -} - -#[test] -fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { - let grove_version = GroveVersion::latest(); - // All elements in a sum tree must have a summed feature type - let db = make_test_grovedb(grove_version); - db.insert( - [TEST_LEAF].as_ref(), - b"key", - Element::empty_sum_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - // Add sum items - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"item1", - Element::new_sum_item(30), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"item2", - Element::new_sum_item(10), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - // Add regular items - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"item3", - Element::new_item(vec![10]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"item4", - Element::new_item(vec![15]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - - let batch = StorageBatch::new(); - - // Open merk and check all elements in it - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert!(matches!( - merk.get_feature_type( - b"item1", - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version + grove_version, ) .unwrap() - .expect("node should exist"), - Some(SummedMerkNode(30)) - )); - assert!(matches!( - merk.get_feature_type( - b"item2", - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"sumkey"].as_ref(), + b"k1", + Element::new_item(vec![1]), + None, + None, + grove_version, ) .unwrap() - .expect("node should exist"), - Some(SummedMerkNode(10)) - )); - assert!(matches!( - merk.get_feature_type( - b"item3", - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"sumkey"].as_ref(), + b"k2", + Element::new_sum_item(5), + None, + None, + grove_version, ) .unwrap() - .expect("node should exist"), - Some(SummedMerkNode(0)) - )); - assert!(matches!( - merk.get_feature_type( - b"item4", - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version - ) - .unwrap() - .expect("node should exist"), - Some(SummedMerkNode(0)) - )); - assert_eq!( - merk.aggregate_data().expect("expected to get sum").as_i64(), - 40 - ); - - // Perform the same test on regular trees - let db = make_test_grovedb(grove_version); - db.insert( - [TEST_LEAF].as_ref(), - b"key", - Element::empty_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"item1", - Element::new_item(vec![30]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"item2", - Element::new_item(vec![10]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert!(matches!( - merk.get_feature_type( - b"item1", - true, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"sumkey"].as_ref(), + b"k3", + Element::empty_tree(), + None, + None, + grove_version, ) .unwrap() - .expect("node should exist"), - Some(BasicMerkNode) - )); - assert!(matches!( - merk.get_feature_type( - b"item2", - true, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version - ) - .unwrap() - .expect("node should exist"), - Some(BasicMerkNode) - )); - assert_eq!( - merk.aggregate_data().expect("expected to get sum"), - AggregateData::NoAggregateData - ); -} + .expect("should insert tree"); -#[test] -fn test_sum_tree_feature() { - let grove_version = GroveVersion::latest(); - let db = make_test_grovedb(grove_version); - db.insert( - [TEST_LEAF].as_ref(), - b"key", - Element::empty_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - - let batch = StorageBatch::new(); - - // Sum should be non for non sum tree - // TODO: change interface to retrieve element directly - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert_eq!( - merk.aggregate_data().expect("expected to get sum"), - AggregateData::NoAggregateData - ); - - // Add sum tree - db.insert( - [TEST_LEAF].as_ref(), - b"key2", - Element::empty_sum_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert sum tree"); - let sum_tree = db - .get([TEST_LEAF].as_ref(), b"key2", None, grove_version) - .unwrap() - .expect("should retrieve tree"); - assert_eq!(sum_tree.sum_value_or_default(), 0); - - // Add sum items to the sum tree - db.insert( - [TEST_LEAF, b"key2"].as_ref(), - b"item1", - Element::new_sum_item(30), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - // TODO: change interface to retrieve element directly - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key2"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert_eq!( - merk.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(30) - ); - - // Add more sum items - db.insert( - [TEST_LEAF, b"key2"].as_ref(), - b"item2", - Element::new_sum_item(-10), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key2"].as_ref(), - b"item3", - Element::new_sum_item(50), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key2"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert_eq!( - merk.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(70) - ); // 30 - 10 + 50 = 70 - - // Add non sum items, result should remain the same - db.insert( - [TEST_LEAF, b"key2"].as_ref(), - b"item4", - Element::new_item(vec![29]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key2"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert_eq!( - merk.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(70) - ); - - // Update existing sum items - db.insert( - [TEST_LEAF, b"key2"].as_ref(), - b"item2", - Element::new_sum_item(10), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key2"].as_ref(), - b"item3", - Element::new_sum_item(-100), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key2"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert_eq!( - merk.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(-60) - ); // 30 + 10 - 100 = -60 - - // We can not replace a normal item with a sum item, so let's delete it first - db.delete( - [TEST_LEAF, b"key2"].as_ref(), - b"item4", - None, - None, - grove_version, - ) - .unwrap() - .expect("expected to delete"); - // Use a large value - db.insert( - [TEST_LEAF, b"key2"].as_ref(), - b"item4", - Element::new_sum_item(10000000), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - let merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key2"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert_eq!( - merk.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(9999940) - ); // 30 + - // 10 - - // 100 + - // 10000000 - - // TODO: Test out overflows -} + // Test proper item retrieval + let item = db + .get([TEST_LEAF, b"sumkey"].as_ref(), b"k2", None, grove_version) + .unwrap() + .expect("should get item"); + assert_eq!(item, Element::new_sum_item(5)); -#[test] -fn test_sum_tree_propagation() { - let grove_version = GroveVersion::latest(); - let db = make_test_grovedb(grove_version); - // Tree - // SumTree - // SumTree - // Item1 - // SumItem1 - // SumItem2 - // SumItem3 - db.insert( - [TEST_LEAF].as_ref(), - b"key", - Element::empty_sum_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"tree2", - Element::empty_sum_tree(), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - db.insert( - [TEST_LEAF, b"key"].as_ref(), - b"sumitem3", - Element::new_sum_item(20), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert tree"); - db.insert( - [TEST_LEAF, b"key", b"tree2"].as_ref(), - b"item1", - Element::new_item(vec![2]), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key", b"tree2"].as_ref(), - b"sumitem1", - Element::new_sum_item(5), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key", b"tree2"].as_ref(), - b"sumitem2", - Element::new_sum_item(10), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - db.insert( - [TEST_LEAF, b"key", b"tree2"].as_ref(), - b"item2", - Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ - TEST_LEAF.to_vec(), - b"key".to_vec(), - b"tree2".to_vec(), - b"sumitem1".to_vec(), - ])), - None, - None, - grove_version, - ) - .unwrap() - .expect("should insert item"); - - let sum_tree = db - .get([TEST_LEAF].as_ref(), b"key", None, grove_version) - .unwrap() - .expect("should fetch tree"); - assert_eq!(sum_tree.sum_value_or_default(), 35); - - let batch = StorageBatch::new(); - - // Assert node feature types - let test_leaf_merk = db - .open_non_transactional_merk_at_path( - [TEST_LEAF].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert!(matches!( - test_leaf_merk - .get_feature_type( - b"key", - true, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version - ) + // Test proof generation + let mut query = Query::new(); + query.insert_key(b"k2".to_vec()); + + let path_query = + PathQuery::new_unsized(vec![TEST_LEAF.to_vec(), b"sumkey".to_vec()], query); + let proof = db + .prove_query(&path_query, None, grove_version) .unwrap() - .expect("node should exist"), - Some(BasicMerkNode) - )); + .expect("should generate proof"); + let (root_hash, result_set) = GroveDb::verify_query_raw(&proof, &path_query, grove_version) + .expect("should verify proof"); + assert_eq!( + root_hash, + db.grove_db.root_hash(None, grove_version).unwrap().unwrap() + ); + assert_eq!(result_set.len(), 1); + let element_from_proof = Element::deserialize(&result_set[0].value, grove_version) + .expect("should deserialize element"); + assert_eq!(element_from_proof, Element::new_sum_item(5)); + assert_eq!(element_from_proof.sum_value_or_default(), 5); + } - let parent_sum_tree = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key"].as_ref().into(), - Some(&batch), + #[test] + fn test_cannot_insert_sum_item_in_regular_tree() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"sumkey", + Element::empty_tree(), + None, + None, grove_version, ) .unwrap() - .expect("should open tree"); - assert!(matches!( - parent_sum_tree - .get_feature_type( - b"tree2", - true, - Some(&Element::value_defined_cost_for_serialized_value), + .expect("should insert tree"); + assert!(matches!( + db.insert( + [TEST_LEAF, b"sumkey"].as_ref(), + b"k1", + Element::new_sum_item(5), + None, + None, grove_version ) - .unwrap() - .expect("node should exist"), - Some(SummedMerkNode(15)) /* 15 because the child sum tree has one sum item of - * value 5 and - * another of value 10 */ - )); + .unwrap(), + Err(Error::InvalidInput("cannot add sum item to non sum tree")) + )); + } - let child_sum_tree = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key", b"tree2"].as_ref().into(), - Some(&batch), + #[test] + fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { + let grove_version = GroveVersion::latest(); + // All elements in a sum tree must have a summed feature type + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + // Add sum items + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item1", + Element::new_sum_item(30), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item2", + Element::new_sum_item(10), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + // Add regular items + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item3", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item4", + Element::new_item(vec![15]), + None, + None, grove_version, ) .unwrap() - .expect("should open tree"); - assert!(matches!( - child_sum_tree - .get_feature_type( + .expect("should insert item"); + + let batch = StorageBatch::new(); + + // Open merk and check all elements in it + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + merk.get_feature_type( b"item1", true, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -756,219 +287,933 @@ fn test_sum_tree_propagation() { ) .unwrap() .expect("node should exist"), - Some(SummedMerkNode(0)) - )); - assert!(matches!( - child_sum_tree - .get_feature_type( - b"sumitem1", + Some(SummedMerkNode(30)) + )); + assert!(matches!( + merk.get_feature_type( + b"item2", true, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version ) .unwrap() .expect("node should exist"), - Some(SummedMerkNode(5)) - )); - assert!(matches!( - child_sum_tree - .get_feature_type( - b"sumitem2", + Some(SummedMerkNode(10)) + )); + assert!(matches!( + merk.get_feature_type( + b"item3", true, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version ) .unwrap() .expect("node should exist"), - Some(SummedMerkNode(10)) - )); - - // TODO: should references take the sum of the referenced element?? - assert!(matches!( - child_sum_tree - .get_feature_type( - b"item2", + Some(SummedMerkNode(0)) + )); + assert!(matches!( + merk.get_feature_type( + b"item4", true, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version ) .unwrap() .expect("node should exist"), - Some(SummedMerkNode(0)) - )); -} + Some(SummedMerkNode(0)) + )); + assert_eq!( + merk.aggregate_data().expect("expected to get sum").as_i64(), + 40 + ); -#[test] -fn test_sum_tree_with_batches() { - let grove_version = GroveVersion::latest(); - let db = make_test_grovedb(grove_version); - let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec()], - b"key1".to_vec(), - Element::empty_sum_tree(), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec()], - b"a".to_vec(), - Element::new_item(vec![214]), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec()], - b"b".to_vec(), - Element::new_sum_item(10), - ), - ]; - db.apply_batch(ops, None, None, grove_version) + // Perform the same test on regular trees + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_tree(), + None, + None, + grove_version, + ) .unwrap() - .expect("should apply batch"); - - let batch = StorageBatch::new(); - let sum_tree = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key1"].as_ref().into(), - Some(&batch), + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item1", + Element::new_item(vec![30]), + None, + None, grove_version, ) .unwrap() - .expect("should open tree"); + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"item2", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); - assert!(matches!( - sum_tree - .get_feature_type( - b"a", + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + merk.get_feature_type( + b"item1", true, Some(&Element::value_defined_cost_for_serialized_value), grove_version ) .unwrap() .expect("node should exist"), - Some(SummedMerkNode(0)) - )); - assert!(matches!( - sum_tree - .get_feature_type( - b"b", + Some(BasicMerkNode) + )); + assert!(matches!( + merk.get_feature_type( + b"item2", true, Some(&Element::value_defined_cost_for_serialized_value), grove_version ) .unwrap() .expect("node should exist"), - Some(SummedMerkNode(10)) - )); + Some(BasicMerkNode) + )); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::NoAggregateData + ); + } - // Create new batch to use existing tree - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec()], - b"c".to_vec(), - Element::new_sum_item(10), - )]; - db.apply_batch(ops, None, None, grove_version) + #[test] + fn test_sum_tree_feature() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_tree(), + None, + None, + grove_version, + ) .unwrap() - .expect("should apply batch"); + .expect("should insert tree"); - let batch = StorageBatch::new(); - let sum_tree = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key1"].as_ref().into(), - Some(&batch), + let batch = StorageBatch::new(); + + // Sum should be non for non sum tree + // TODO: change interface to retrieve element directly + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::NoAggregateData + ); + + // Add sum tree + db.insert( + [TEST_LEAF].as_ref(), + b"key2", + Element::empty_sum_tree(), + None, + None, grove_version, ) .unwrap() - .expect("should open tree"); - assert!(matches!( - sum_tree - .get_feature_type( - b"c", - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version + .expect("should insert sum tree"); + let sum_tree = db + .get([TEST_LEAF].as_ref(), b"key2", None, grove_version) + .unwrap() + .expect("should retrieve tree"); + assert_eq!(sum_tree.sum_value_or_default(), 0); + + // Add sum items to the sum tree + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item1", + Element::new_sum_item(30), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + // TODO: change interface to retrieve element directly + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, ) .unwrap() - .expect("node should exist"), - Some(SummedMerkNode(10)) - )); - assert_eq!( - sum_tree.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(20) - ); - - // Test propagation - // Add a new sum tree with its own sum items, should affect sum of original - // tree - let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec()], - b"d".to_vec(), + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(30) + ); + + // Add more sum items + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item2", + Element::new_sum_item(-10), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item3", + Element::new_sum_item(50), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(70) + ); // 30 - 10 + 50 = 70 + + // Add non sum items, result should remain the same + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item4", + Element::new_item(vec![29]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(70) + ); + + // Update existing sum items + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item2", + Element::new_sum_item(10), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item3", + Element::new_sum_item(-100), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(-60) + ); // 30 + 10 - 100 = -60 + + // We can not replace a normal item with a sum item, so let's delete it first + db.delete( + [TEST_LEAF, b"key2"].as_ref(), + b"item4", + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to delete"); + // Use a large value + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item4", + Element::new_sum_item(10000000), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(9999940) + ); // 30 + + // 10 - + // 100 + + // 10000000 + } + + #[test] + fn test_sum_tree_overflow() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + db.insert( + [TEST_LEAF].as_ref(), + b"key", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + + let batch = StorageBatch::new(); + + // Sum should be non for non sum tree + // TODO: change interface to retrieve element directly + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::NoAggregateData + ); + + // Add sum tree + db.insert( + [TEST_LEAF].as_ref(), + b"key2", Element::empty_sum_tree(), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], - b"first".to_vec(), - Element::new_sum_item(4), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], - b"second".to_vec(), - Element::new_item(vec![4]), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec()], - b"e".to_vec(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert sum tree"); + let sum_tree = db + .get([TEST_LEAF].as_ref(), b"key2", None, grove_version) + .unwrap() + .expect("should retrieve tree"); + assert_eq!(sum_tree.sum_value_or_default(), 0); + + // Add sum items to the sum tree + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item1", + Element::new_sum_item(SumValue::MAX), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + // TODO: change interface to retrieve element directly + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(SumValue::MAX) + ); + + // Subtract 10 from Max should work + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item2", + Element::new_sum_item(-10), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(SumValue::MAX - 10) + ); + + // Add 20 from Max should overflow + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item3", + Element::new_sum_item(20), + None, + None, + grove_version, + ) + .unwrap() + .expect_err("should not be able to insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(SumValue::MAX - 10) + ); + + // Add non sum items, result should remain the same + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item4", + Element::new_item(vec![29]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(SumValue::MAX - 10) + ); + + // Update existing sum item will overflow + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item2", + Element::new_sum_item(10), // we are replacing -10 with 10 + None, + None, + grove_version, + ) + .unwrap() + .expect_err("should not be able to insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(SumValue::MAX - 10) + ); + + // Update existing sum item will overflow + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item2", + Element::new_sum_item(SumValue::MIN), // we are replacing -10 with SumValue::MIN + None, + None, + grove_version, + ) + .unwrap() + .expect("should be able to insert item"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(-1) + ); + + db.insert( + [TEST_LEAF, b"key2"].as_ref(), + b"item3", + Element::new_sum_item(-40), + None, + None, + grove_version, + ) + .unwrap() + .expect("should be able to insert item"); + + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(-41) + ); + + // Deleting item1 should make us overflow + db.delete( + [TEST_LEAF, b"key2"].as_ref(), + b"item1", + None, + None, + grove_version, + ) + .unwrap() + .expect_err("expected not be able to delete"); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(-41) + ); + } + + #[test] + fn test_sum_tree_propagation() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + // Tree + // SumTree + // SumTree + // Item1 + // SumItem1 + // SumItem2 + // SumItem3 + db.insert( + [TEST_LEAF].as_ref(), + b"key", Element::empty_sum_tree(), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], - b"first".to_vec(), - Element::new_sum_item(12), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], - b"second".to_vec(), - Element::new_item(vec![4]), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], - b"third".to_vec(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"tree2", Element::empty_sum_tree(), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![ - TEST_LEAF.to_vec(), - b"key1".to_vec(), - b"e".to_vec(), - b"third".to_vec(), - ], - b"a".to_vec(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"key"].as_ref(), + b"sumitem3", + Element::new_sum_item(20), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"key", b"tree2"].as_ref(), + b"item1", + Element::new_item(vec![2]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key", b"tree2"].as_ref(), + b"sumitem1", Element::new_sum_item(5), - ), - QualifiedGroveDbOp::insert_or_replace_op( - vec![ + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key", b"tree2"].as_ref(), + b"sumitem2", + Element::new_sum_item(10), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"key", b"tree2"].as_ref(), + b"item2", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ TEST_LEAF.to_vec(), + b"key".to_vec(), + b"tree2".to_vec(), + b"sumitem1".to_vec(), + ])), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + let sum_tree = db + .get([TEST_LEAF].as_ref(), b"key", None, grove_version) + .unwrap() + .expect("should fetch tree"); + assert_eq!(sum_tree.sum_value_or_default(), 35); + + let batch = StorageBatch::new(); + + // Assert node feature types + let test_leaf_merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + test_leaf_merk + .get_feature_type( + b"key", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(BasicMerkNode) + )); + + let parent_sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + parent_sum_tree + .get_feature_type( + b"tree2", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(15)) /* 15 because the child sum tree has one sum item of + * value 5 and + * another of value 10 */ + )); + + let child_sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key", b"tree2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + child_sum_tree + .get_feature_type( + b"item1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(0)) + )); + assert!(matches!( + child_sum_tree + .get_feature_type( + b"sumitem1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(5)) + )); + assert!(matches!( + child_sum_tree + .get_feature_type( + b"sumitem2", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(10)) + )); + + // TODO: should references take the sum of the referenced element?? + assert!(matches!( + child_sum_tree + .get_feature_type( + b"item2", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(0)) + )); + } + + #[test] + fn test_sum_tree_with_batches() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], b"key1".to_vec(), + Element::empty_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"a".to_vec(), + Element::new_item(vec![214]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"b".to_vec(), + Element::new_sum_item(10), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + + assert!(matches!( + sum_tree + .get_feature_type( + b"a", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(0)) + )); + assert!(matches!( + sum_tree + .get_feature_type( + b"b", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(10)) + )); + + // Create new batch to use existing tree + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"c".to_vec(), + Element::new_sum_item(10), + )]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + sum_tree + .get_feature_type( + b"c", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(10)) + )); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(20) + ); + + // Test propagation + // Add a new sum tree with its own sum items, should affect sum of original + // tree + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"d".to_vec(), + Element::empty_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], + b"first".to_vec(), + Element::new_sum_item(4), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], + b"second".to_vec(), + Element::new_item(vec![4]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"e".to_vec(), + Element::empty_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"first".to_vec(), + Element::new_sum_item(12), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"second".to_vec(), + Element::new_item(vec![4]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"third".to_vec(), - ], - b"b".to_vec(), - Element::new_item(vec![5]), - ), - ]; - db.apply_batch(ops, None, None, grove_version) - .unwrap() - .expect("should apply batch"); + Element::empty_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![ + TEST_LEAF.to_vec(), + b"key1".to_vec(), + b"e".to_vec(), + b"third".to_vec(), + ], + b"a".to_vec(), + Element::new_sum_item(5), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![ + TEST_LEAF.to_vec(), + b"key1".to_vec(), + b"e".to_vec(), + b"third".to_vec(), + ], + b"b".to_vec(), + Element::new_item(vec![5]), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); - let batch = StorageBatch::new(); - let sum_tree = db - .open_non_transactional_merk_at_path( - [TEST_LEAF, b"key1"].as_ref().into(), - Some(&batch), - grove_version, - ) - .unwrap() - .expect("should open tree"); - assert_eq!( - sum_tree.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(41) - ); + let batch = StorageBatch::new(); + let sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(41) + ); + } } diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index f5a8fdd37..9ff58aef5 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -2,10 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::{CostResult, CostsExt, OperationCost}; -use grovedb_version::{ - error::GroveVersionError, - version::{FeatureVersion, GroveVersion}, -}; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 2d24825eb..7600246be 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -458,7 +458,7 @@ impl TreeNode { match link.aggregateData() { AggregateData::NoAggregateData => 0, AggregateData::Sum(s) => s.encode_var_vec().len() as u32, - AggregateData::BigSum(s) => 16 as u32, + AggregateData::BigSum(_) => 16 as u32, AggregateData::Count(c) => c.encode_var_vec().len() as u32, }, ) @@ -506,7 +506,7 @@ impl TreeNode { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => Ok(0), AggregateData::Sum(s) => Ok(s), - AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree( + AggregateData::BigSum(_) => Err(Error::BigSumTreeUnderNormalSumTree( "for aggregate data as i64".to_string(), )), AggregateData::Count(c) => { @@ -535,7 +535,7 @@ impl TreeNode { Ok(s as u64) } } - AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree( + AggregateData::BigSum(_) => Err(Error::BigSumTreeUnderNormalSumTree( "for aggregate data as u64".to_string(), )), AggregateData::Count(c) => Ok(c), From 3f6705ff5abba19a7a617c8f972af7ef8c0e1872 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 9 Jan 2025 06:01:53 +0700 Subject: [PATCH 07/30] added count sum tree --- .../estimated_costs/average_case_costs.rs | 1 - grovedb/src/batch/mod.rs | 21 +- grovedb/src/element/constructor.rs | 26 + grovedb/src/element/delete.rs | 18 +- grovedb/src/element/get.rs | 6 +- grovedb/src/element/helpers.rs | 32 +- grovedb/src/element/mod.rs | 21 + grovedb/src/lib.rs | 14 +- grovedb/src/operations/get/mod.rs | 2 +- grovedb/src/operations/get/query.rs | 15 +- grovedb/src/operations/proof/verify.rs | 4 +- grovedb/src/tests/count_tree_tests.rs | 10 +- grovedb/src/tests/sum_tree_tests.rs | 453 +++++++++++++++++- grovedb/src/visualize.rs | 11 + merk/src/estimated_costs/mod.rs | 9 + merk/src/merk/committer.rs | 2 +- merk/src/merk/mod.rs | 9 + merk/src/tree/link.rs | 30 +- merk/src/tree/mod.rs | 61 +-- merk/src/tree/tree_feature_type.rs | 34 +- node-grove/src/converter.rs | 4 +- 21 files changed, 703 insertions(+), 80 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 93c6dccda..112aac040 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -13,7 +13,6 @@ use grovedb_costs::{ #[cfg(feature = "full")] use grovedb_merk::{ estimated_costs::average_case_costs::{average_case_merk_propagate, EstimatedLayerInformation}, - IsSumTree, }; use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index c14426ef7..1d4316242 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -75,7 +75,7 @@ use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, element::{ - MaxReferenceHop, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, + MaxReferenceHop, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE, }, operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, @@ -1022,7 +1022,7 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + | Element::CountTree(..) | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1143,7 +1143,7 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + | Element::CountTree(..) | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1173,7 +1173,7 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + | Element::CountTree(..) | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1348,7 +1348,7 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => { + | Element::CountTree(..) | Element::CountSumTree(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element @@ -1562,6 +1562,14 @@ where flags, ) } + AggregateData::CountAndSum(count_value, sum_value) => { + Element::new_count_sum_tree_with_flags_and_sum_and_count_value( + root_key, + count_value, + sum_value, + flags, + ) + } }; let merk_feature_type = cost_return_on_error_no_add!(&cost, element.get_feature_type(in_tree_type)); @@ -1650,13 +1658,14 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => { + | Element::CountTree(..) | Element::CountSumTree(..) => { let tree_type = new_element.tree_type().unwrap(); let tree_cost_size = match tree_type { TreeType::NormalTree => TREE_COST_SIZE, TreeType::SumTree => SUM_TREE_COST_SIZE, TreeType::BigSumTree => BIG_SUM_TREE_COST_SIZE, TreeType::CountTree => COUNT_TREE_COST_SIZE, + TreeType::CountSumTree => COUNT_SUM_TREE_COST_SIZE, }; let tree_value_cost = tree_cost_size + flags_len diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 0ba0fd8ff..3dc346b3b 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -194,4 +194,30 @@ impl Element { ) -> Self { Element::CountTree(maybe_root_key, count_value, flags) } + + #[cfg(feature = "full")] + /// Set element to a count sum tree without flags + pub fn new_count_sum_tree(maybe_root_key: Option>) -> Self { + Element::CountSumTree(maybe_root_key, 0, 0, None) + } + + #[cfg(feature = "full")] + /// Set element to a count sum tree with flags + pub fn new_count_sum_tree_with_flags( + maybe_root_key: Option>, + flags: Option, + ) -> Self { + Element::CountSumTree(maybe_root_key, 0, 0, flags) + } + + #[cfg(feature = "full")] + /// Set element to a count sum tree with flags and sum value + pub fn new_count_sum_tree_with_flags_and_sum_and_count_value( + maybe_root_key: Option>, + count_value: CountValue, + sum_value: SumValue, + flags: Option, + ) -> Self { + Element::CountSumTree(maybe_root_key, count_value, sum_value, flags) + } } diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index 2ab01e02f..41feb6400 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -37,10 +37,12 @@ impl Element { (TreeType::NormalTree, false) => Op::Delete, (TreeType::SumTree, true) | (TreeType::BigSumTree, true) - | (TreeType::CountTree, true) => Op::DeleteLayeredMaybeSpecialized, + | (TreeType::CountTree, true) + | (TreeType::CountSumTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) - | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, + | (TreeType::CountTree, false) + | (TreeType::CountSumTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; // todo not sure we get it again, we need to see if this is necessary @@ -94,10 +96,12 @@ impl Element { (TreeType::NormalTree, false) => Op::Delete, (TreeType::SumTree, true) | (TreeType::BigSumTree, true) - | (TreeType::CountTree, true) => Op::DeleteLayeredMaybeSpecialized, + | (TreeType::CountTree, true) + | (TreeType::CountSumTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) - | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, + | (TreeType::CountTree, false) + | (TreeType::CountSumTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; // todo not sure we get it again, we need to see if this is necessary @@ -145,10 +149,12 @@ impl Element { (TreeType::NormalTree, false) => Op::Delete, (TreeType::SumTree, true) | (TreeType::BigSumTree, true) - | (TreeType::CountTree, true) => Op::DeleteLayeredMaybeSpecialized, + | (TreeType::CountTree, true) + | (TreeType::CountSumTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) - | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, + | (TreeType::CountTree, false) + | (TreeType::CountSumTree, false) => Op::DeleteMaybeSpecialized, }; let entry = (key, op); batch_operations.push(entry); diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index f61a20d61..9ff281daa 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -200,7 +200,8 @@ impl Element { Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags)) | Some(Element::BigSumTree(_, _, flags)) - | Some(Element::CountTree(_, _, flags)) => { + | Some(Element::CountTree(_, _, flags)) + | Some(Element::CountSumTree(.. , flags)) => { let tree_cost_size = element.as_ref().unwrap().tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -281,7 +282,8 @@ impl Element { Element::Tree(_, flags) | Element::SumTree(_, _, flags) | Element::BigSumTree(_, _, flags) - | Element::CountTree(_, _, flags) => { + | Element::CountTree(_, _, flags) + | Element::CountSumTree(.., flags) => { let tree_cost_size = element.tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 8b8a22a18..0acb9d3f4 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -20,8 +20,8 @@ use grovedb_merk::{ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; - -use crate::element::{BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE}; +use grovedb_merk::TreeFeatureType::CountedSummedMerkNode; +use crate::element::{BIG_SUM_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE}; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; #[cfg(any(feature = "full", feature = "verify"))] @@ -183,6 +183,7 @@ impl Element { Element::SumTree(..) => Some(TreeType::SumTree), Element::BigSumTree(..) => Some(TreeType::BigSumTree), Element::CountTree(..) => Some(TreeType::CountTree), + Element::CountSumTree(..) => Some(TreeType::CountSumTree), _ => None, } } @@ -208,6 +209,7 @@ impl Element { | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..) + | Element::CountSumTree(..) ) } @@ -243,6 +245,7 @@ impl Element { TreeType::SumTree => Ok(SummedMerkNode(self.sum_value_or_default())), TreeType::BigSumTree => Ok(BigSummedMerkNode(self.big_sum_value_or_default())), TreeType::CountTree => Ok(CountedMerkNode(self.count_value_or_default())), + TreeType::CountSumTree => Ok(CountedSummedMerkNode(self.count_value_or_default(), self.sum_value_or_default())), } } @@ -256,7 +259,8 @@ impl Element { | Element::SumTree(.., flags) | Element::BigSumTree(.., flags) | Element::CountTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::CountSumTree(.., flags) => flags, } } @@ -270,7 +274,8 @@ impl Element { | Element::SumTree(.., flags) | Element::BigSumTree(.., flags) | Element::CountTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::CountSumTree(.., flags) => flags, } } @@ -284,7 +289,8 @@ impl Element { | Element::SumTree(.., flags) | Element::BigSumTree(.., flags) | Element::CountTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::CountSumTree(.., flags) => flags, } } @@ -298,7 +304,8 @@ impl Element { | Element::SumTree(.., flags) | Element::BigSumTree(.., flags) | Element::CountTree(.., flags) - | Element::SumItem(_, flags) => *flags = new_flags, + | Element::SumItem(_, flags) + | Element::CountSumTree(.., flags) => *flags = new_flags, } } @@ -409,6 +416,17 @@ impl Element { key_len, value_len, node_type, ) } + Element::CountSumTree(.. , flags) => { + let flags_len = flags.map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let value_len = COUNT_SUM_TREE_COST_SIZE + flags_len; + let key_len = key.len() as u32; + KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key_len, value_len, node_type, + ) + } Element::SumItem(.., flags) => { let flags_len = flags.map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -436,6 +454,7 @@ impl Element { Element::BigSumTree(..) => Ok(BIG_SUM_TREE_COST_SIZE), Element::SumItem(..) => Ok(SUM_ITEM_COST_SIZE), Element::CountTree(..) => Ok(COUNT_TREE_COST_SIZE), + Element::CountSumTree(..) => Ok(COUNT_SUM_TREE_COST_SIZE), _ => Err(Error::CorruptedCodeExecution( "trying to get tree cost from non tree element", )), @@ -459,6 +478,7 @@ impl Element { Element::SumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::BigSumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::CountTree(..) => Some(LayeredValueDefinedCost(cost)), + Element::CountSumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::SumItem(..) => Some(SpecializedValueDefinedCost(cost)), _ => None, } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 2784bb98f..5b831a11b 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -30,6 +30,7 @@ use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; use grovedb_merk::estimated_costs::{ BIG_SUM_LAYER_COST_SIZE, LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, }; +use grovedb_merk::estimated_costs::SUM_AND_COUNT_LAYER_COST_SIZE; #[cfg(feature = "full")] use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] @@ -72,6 +73,10 @@ pub const BIG_SUM_TREE_COST_SIZE: u32 = BIG_SUM_LAYER_COST_SIZE; // 19 /// The cost of a count tree pub const COUNT_TREE_COST_SIZE: u32 = SUM_LAYER_COST_SIZE; // 12 +#[cfg(feature = "full")] +/// The cost of a count tree +pub const COUNT_SUM_TREE_COST_SIZE: u32 = SUM_AND_COUNT_LAYER_COST_SIZE; // 21 + #[cfg(any(feature = "full", feature = "verify"))] /// int 64 sum value pub type SumValue = i64; @@ -95,6 +100,7 @@ impl CostSize for TreeType { TreeType::SumTree => SUM_TREE_COST_SIZE, TreeType::BigSumTree => BIG_SUM_TREE_COST_SIZE, TreeType::CountTree => COUNT_TREE_COST_SIZE, + TreeType::CountSumTree => COUNT_SUM_TREE_COST_SIZE, } } } @@ -127,6 +133,8 @@ pub enum Element { /// Same as Element::Tree but underlying Merk counts value of its countable /// nodes CountTree(Option>, CountValue, Option), + /// Combines Element::SumTree and Element::CountTree + CountSumTree(Option>, CountValue, SumValue, Option), } impl fmt::Display for Element { @@ -206,6 +214,18 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::CountSumTree(root_key, count_value, sum_value, flags) => { + write!( + f, + "CountSumTree({}, {}, {}{})", + root_key.as_ref().map_or("None".to_string(), hex::encode), + count_value, + sum_value, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } } } } @@ -220,6 +240,7 @@ impl Element { Element::SumTree(..) => "sum tree", Element::BigSumTree(..) => "big sum tree", Element::CountTree(..) => "count tree", + Element::CountSumTree(..) => "count sum tree", } } diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 36907b094..a757dade4 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -810,7 +810,7 @@ impl GroveDb { } else if let Element::SumTree(.., flag) = element { let tree = Element::new_sum_tree_with_flags_and_sum_value( maybe_root_key, - aggregate_data.as_i64(), + aggregate_data.as_sum_i64(), flag, ); tree.insert_subtree( @@ -836,7 +836,7 @@ impl GroveDb { } else if let Element::CountTree(.., flag) = element { let tree = Element::new_count_tree_with_flags_and_count_value( maybe_root_key, - aggregate_data.as_u64(), + aggregate_data.as_count_u64(), flag, ); tree.insert_subtree( @@ -891,7 +891,7 @@ impl GroveDb { } else if let Element::SumTree(.., flag) = element { let tree = Element::new_sum_tree_with_flags_and_sum_value( maybe_root_key, - aggregate_data.as_i64(), + aggregate_data.as_sum_i64(), flag, ); let merk_feature_type = cost_return_on_error!( @@ -929,7 +929,7 @@ impl GroveDb { } else if let Element::CountTree(.., flag) = element { let tree = Element::new_count_tree_with_flags_and_count_value( maybe_root_key, - aggregate_data.as_u64(), + aggregate_data.as_count_u64(), flag, ); let merk_feature_type = cost_return_on_error!( @@ -1177,7 +1177,8 @@ impl GroveDb { Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => { + | Element::CountTree(..) + | Element::CountSumTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -1324,7 +1325,8 @@ impl GroveDb { Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => { + | Element::CountTree(..) + | Element::CountSumTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index 073e02abe..c035147f0 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -458,7 +458,7 @@ impl GroveDb { Ok(Element::Tree(..)) | Ok(Element::SumTree(..)) | Ok(Element::BigSumTree(..)) - | Ok(Element::CountTree(..)) => Ok(()).wrap_with_cost(cost), + | Ok(Element::CountTree(..)) | Ok(Element::CountSumTree(..)) => Ok(()).wrap_with_cost(cost), Ok(_) | Err(Error::PathKeyNotFound(_)) => Err(error_fn()).wrap_with_cost(cost), Err(e) => Err(e).wrap_with_cost(cost), } diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 14ed383b6..584e6ce7e 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -37,6 +37,8 @@ pub enum QueryItemOrSumReturnType { BigSumValue(BigSumValue), /// A count value CountValue(CountValue), + /// A count and sum value + CountSumValue(CountValue, SumValue), } #[cfg(feature = "full")] @@ -231,7 +233,8 @@ where { | Element::SumItem(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => Ok(element), + | Element::CountTree(..) + | Element::CountSumTree(..) => Ok(element), Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -353,7 +356,8 @@ where { Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) => Err(Error::InvalidQuery( + | Element::CountTree(..) + | Element::CountSumTree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items and references", )), } @@ -440,6 +444,9 @@ where { Element::CountTree(_, count_value, _) => { Ok(QueryItemOrSumReturnType::CountValue(count_value)) } + Element::CountSumTree(_, count_value, sum_value, _) => { + Ok(QueryItemOrSumReturnType::CountSumValue(count_value, sum_value)) + } _ => Err(Error::InvalidQuery( "the reference must result in an item", )), @@ -463,6 +470,9 @@ where { Element::CountTree(_, count_value, _) => { Ok(QueryItemOrSumReturnType::CountValue(count_value)) } + Element::CountSumTree(_, count_value, sum_value, _) => { + Ok(QueryItemOrSumReturnType::CountSumValue(count_value, sum_value)) + } Element::Tree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items, sum items, references and sum \ trees", @@ -548,6 +558,7 @@ where { | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) + | Element::CountSumTree(..) | Element::Item(..) => Err(Error::InvalidQuery( "path_queries over sum items can only refer to sum items and \ references", diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 0130bac62..dca00bb93 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -311,7 +311,8 @@ impl GroveDb { Element::Tree(Some(_), _) | Element::SumTree(Some(_), ..) | Element::BigSumTree(Some(_), ..) - | Element::CountTree(Some(_), ..) => { + | Element::CountTree(Some(_), ..) + | Element::CountSumTree(Some(_), ..) => { path.push(key); let lower_hash = Self::verify_layer_proof( lower_layer, @@ -342,6 +343,7 @@ impl GroveDb { | Element::SumTree(None, ..) | Element::BigSumTree(None, ..) | Element::CountTree(None, ..) + | Element::CountSumTree(None, ..) | Element::SumItem(..) | Element::Item(..) | Element::Reference(..) => { diff --git a/grovedb/src/tests/count_tree_tests.rs b/grovedb/src/tests/count_tree_tests.rs index 6ac71ed02..e4dffc06c 100644 --- a/grovedb/src/tests/count_tree_tests.rs +++ b/grovedb/src/tests/count_tree_tests.rs @@ -15,7 +15,7 @@ mod tests { batch::QualifiedGroveDbOp, reference_path::ReferencePathType, tests::{make_test_grovedb, TEST_LEAF}, - Element, Error, GroveDb, PathQuery, + Element, GroveDb, PathQuery, }; #[test] @@ -216,10 +216,10 @@ mod tests { .expect("node should exist") .expect("expected feature type"); - assert_matches!(feature_type_node_1, CountedMerkNode(1)); - assert_matches!(feature_type_node_2, CountedMerkNode(1)); - assert_matches!(feature_type_node_3, CountedMerkNode(1)); - assert_matches!(feature_type_node_4, CountedMerkNode(1)); + assert_eq!(feature_type_node_1, CountedMerkNode(1)); + assert_eq!(feature_type_node_2, CountedMerkNode(1)); + assert_eq!(feature_type_node_3, CountedMerkNode(1)); + assert_eq!(feature_type_node_4, CountedMerkNode(1)); // Perform the same test on regular trees let db = make_test_grovedb(grove_version); diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 215249ca1..0c9c19dbc 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -7,6 +7,7 @@ mod tests { tree::{kv::ValueDefinedCostType, AggregateData}, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; + use grovedb_merk::TreeFeatureType::BigSummedMerkNode; use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; @@ -323,7 +324,7 @@ mod tests { Some(SummedMerkNode(0)) )); assert_eq!( - merk.aggregate_data().expect("expected to get sum").as_i64(), + merk.aggregate_data().expect("expected to get sum").as_sum_i64(), 40 ); @@ -1042,6 +1043,269 @@ mod tests { )); } + #[test] + fn test_big_sum_tree_propagation() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + // Tree + // BigSumTree + // SumTree1 + // SumItem1 + // SumItem2 + // SumTree2 + // SumItem3 + // SumItem4 + db.insert( + [TEST_LEAF].as_ref(), + b"big_sum_tree", + Element::empty_big_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"big_sum_tree"].as_ref(), + b"sum_tree_1", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"big_sum_tree"].as_ref(), + b"sum_tree_2", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert tree"); + db.insert( + [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), + b"item1", + Element::new_item(vec![2]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), + b"sum_item_1", + Element::new_sum_item(SumValue::MAX - 40), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), + b"sum_item_2", + Element::new_sum_item(30), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + db.insert( + [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), + b"ref_1", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"big_sum_tree".to_vec(), + b"sum_tree_1".to_vec(), + b"sum_item_1".to_vec(), + ])), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + db.insert( + [TEST_LEAF, b"big_sum_tree", b"sum_tree_2"].as_ref(), + b"sum_item_3", + Element::new_sum_item(SumValue::MAX - 50), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + let sum_tree = db + .get([TEST_LEAF].as_ref(), b"big_sum_tree", None, grove_version) + .unwrap() + .expect("should fetch tree"); + assert_eq!(sum_tree.big_sum_value_or_default(), (SumValue::MAX - 10) as i128 + (SumValue::MAX - 50) as i128); + + db.insert( + [TEST_LEAF, b"big_sum_tree"].as_ref(), + b"sum_item_4", + Element::new_sum_item(SumValue::MAX - 70), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item"); + + let sum_tree = db + .get([TEST_LEAF].as_ref(), b"big_sum_tree", None, grove_version) + .unwrap() + .expect("should fetch tree"); + assert_eq!(sum_tree.big_sum_value_or_default(), (SumValue::MAX - 10) as i128 + (SumValue::MAX - 50) as i128 + (SumValue::MAX - 70) as i128 ); + + let batch = StorageBatch::new(); + + // Assert node feature types + let test_leaf_merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert!(matches!( + test_leaf_merk + .get_feature_type( + b"big_sum_tree", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(BasicMerkNode) + )); + + let parent_sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"big_sum_tree"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + let feature_type = parent_sum_tree + .get_feature_type( + b"sum_tree_1", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist").expect("expected feature type"); + assert_eq!( + feature_type, + BigSummedMerkNode((SumValue::MAX - 10) as i128) + ); + + let feature_type = parent_sum_tree + .get_feature_type( + b"sum_item_4", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist").expect("expected feature type"); + assert_eq!( + feature_type, + BigSummedMerkNode((SumValue::MAX - 70) as i128) + ); + + let child_sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + child_sum_tree + .get_feature_type( + b"item1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(0)) + ); + assert_eq!( + child_sum_tree + .get_feature_type( + b"sum_item_1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(SumValue::MAX - 40)) + ); + assert_eq!( + child_sum_tree + .get_feature_type( + b"sum_item_2", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(30)) + ); + + assert_eq!( + child_sum_tree + .get_feature_type( + b"ref_1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(0)) + ); + + let child_sum_tree_2 = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"big_sum_tree", b"sum_tree_2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + + assert_eq!( + child_sum_tree_2 + .get_feature_type( + b"sum_item_3", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(SumValue::MAX - 50)) + ); + } + #[test] fn test_sum_tree_with_batches() { let grove_version = GroveVersion::latest(); @@ -1077,7 +1341,7 @@ mod tests { .unwrap() .expect("should open tree"); - assert!(matches!( + assert_eq!( sum_tree .get_feature_type( b"a", @@ -1088,8 +1352,8 @@ mod tests { .unwrap() .expect("node should exist"), Some(SummedMerkNode(0)) - )); - assert!(matches!( + ); + assert_eq!( sum_tree .get_feature_type( b"b", @@ -1100,7 +1364,7 @@ mod tests { .unwrap() .expect("node should exist"), Some(SummedMerkNode(10)) - )); + ); // Create new batch to use existing tree let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( @@ -1121,7 +1385,7 @@ mod tests { ) .unwrap() .expect("should open tree"); - assert!(matches!( + assert_eq!( sum_tree .get_feature_type( b"c", @@ -1132,7 +1396,182 @@ mod tests { .unwrap() .expect("node should exist"), Some(SummedMerkNode(10)) - )); + ); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(20) + ); + + // Test propagation + // Add a new sum tree with its own sum items, should affect sum of original + // tree + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"d".to_vec(), + Element::empty_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], + b"first".to_vec(), + Element::new_sum_item(4), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], + b"second".to_vec(), + Element::new_item(vec![4]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"e".to_vec(), + Element::empty_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"first".to_vec(), + Element::new_sum_item(12), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"second".to_vec(), + Element::new_item(vec![4]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], + b"third".to_vec(), + Element::empty_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![ + TEST_LEAF.to_vec(), + b"key1".to_vec(), + b"e".to_vec(), + b"third".to_vec(), + ], + b"a".to_vec(), + Element::new_sum_item(5), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![ + TEST_LEAF.to_vec(), + b"key1".to_vec(), + b"e".to_vec(), + b"third".to_vec(), + ], + b"b".to_vec(), + Element::new_item(vec![5]), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(41) + ); + } + + #[test] + fn test_big_sum_tree_with_batches() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"key1".to_vec(), + Element::empty_big_sum_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"a".to_vec(), + Element::new_item(vec![214]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"b".to_vec(), + Element::new_sum_item(10), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + + assert_eq!( + sum_tree + .get_feature_type( + b"a", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(BigSummedMerkNode(0)) + ); + assert_eq!( + sum_tree + .get_feature_type( + b"b", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(BigSummedMerkNode(10)) + ); + + // Create new batch to use existing tree + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"key1".to_vec()], + b"c".to_vec(), + Element::new_sum_item(10), + )]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + let batch = StorageBatch::new(); + let sum_tree = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"key1"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open tree"); + assert_eq!( + sum_tree + .get_feature_type( + b"c", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(10)) + ); assert_eq!( sum_tree.aggregate_data().expect("expected to get sum"), AggregateData::Sum(20) diff --git a/grovedb/src/visualize.rs b/grovedb/src/visualize.rs index 0659db9a3..8fdccc7e0 100644 --- a/grovedb/src/visualize.rs +++ b/grovedb/src/visualize.rs @@ -117,6 +117,17 @@ impl Visualize for Element { drawer = root_key.as_deref().visualize(drawer)?; drawer.write(format!(" {value}").as_bytes())?; + if let Some(f) = flags { + if !f.is_empty() { + drawer = f.visualize(drawer)?; + } + } + } + Element::CountSumTree(root_key, count_value, sum_value, flags) => { + drawer.write(b"count_sum_tree: ")?; + drawer = root_key.as_deref().visualize(drawer)?; + drawer.write(format!("count: {count_value}, sum {sum_value}").as_bytes())?; + if let Some(f) = flags { if !f.is_empty() { drawer = f.visualize(drawer)?; diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index 5d90f0f63..eb2eaa404 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -27,6 +27,10 @@ pub const LAYER_COST_SIZE: u32 = 3; /// The cost of a sum value pub const SUM_VALUE_EXTRA_COST: u32 = 9; +#[cfg(any(feature = "full", feature = "verify"))] +/// The cost of a count value +pub const COUNT_VALUE_EXTRA_COST: u32 = 9; + #[cfg(any(feature = "full", feature = "verify"))] /// The cost of a big sum value pub const BIG_SUM_VALUE_EXTRA_COST: u32 = 16; @@ -36,6 +40,11 @@ pub const BIG_SUM_VALUE_EXTRA_COST: u32 = 16; /// This is the layer size + 9 for the encoded value pub const SUM_LAYER_COST_SIZE: u32 = LAYER_COST_SIZE + SUM_VALUE_EXTRA_COST; +#[cfg(feature = "full")] +/// The cost of a summed subtree layer +/// This is the layer size + 9 for the encoded value +pub const SUM_AND_COUNT_LAYER_COST_SIZE: u32 = LAYER_COST_SIZE + SUM_VALUE_EXTRA_COST + COUNT_VALUE_EXTRA_COST; + #[cfg(feature = "full")] /// The cost of a summed subtree layer /// This is the layer size + 16 for the encoded value diff --git a/merk/src/merk/committer.rs b/merk/src/merk/committer.rs index 9fb029875..c260c3556 100644 --- a/merk/src/merk/committer.rs +++ b/merk/src/merk/committer.rs @@ -44,7 +44,7 @@ impl Commit for MerkCommitter { let right_child_sizes = tree.child_ref_and_sum_size(false); self.batch.push(( tree.key().to_vec(), - tree.feature_type().sum_length(), + tree.feature_type().tree_feature_length(), Some((buf, left_child_sizes, right_child_sizes)), storage_costs, )); diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index eadbe6966..e4abe7508 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -250,6 +250,7 @@ pub enum TreeType { SumTree = 1, BigSumTree = 2, CountTree = 3, + CountSumTree = 4, } impl TryFrom for TreeType { @@ -261,6 +262,7 @@ impl TryFrom for TreeType { 1 => Ok(TreeType::SumTree), 2 => Ok(TreeType::BigSumTree), 3 => Ok(TreeType::CountTree), + 3 => Ok(TreeType::CountSumTree), n => Err(Error::UnknownTreeType(format!("got {}, max is 3", n))), // Error handling } } @@ -273,6 +275,7 @@ impl fmt::Display for TreeType { TreeType::SumTree => "Sum Tree", TreeType::BigSumTree => "Big Sum Tree", TreeType::CountTree => "Count Tree", + TreeType::CountSumTree => "Count Sum Tree", }; write!(f, "{}", s) } @@ -285,6 +288,7 @@ impl TreeType { TreeType::SumTree => true, TreeType::BigSumTree => true, TreeType::CountTree => false, + TreeType::CountSumTree => true, } } @@ -294,6 +298,7 @@ impl TreeType { TreeType::SumTree => NodeType::SumNode, TreeType::BigSumTree => NodeType::BigSumNode, TreeType::CountTree => NodeType::CountNode, + TreeType::CountSumTree => NodeType::CountSumNode, } } @@ -303,6 +308,7 @@ impl TreeType { TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), + TreeType::CountSumTree => TreeFeatureType::CountedSummedMerkNode(0, 0), } } } @@ -314,6 +320,7 @@ pub enum NodeType { SumNode, BigSumNode, CountNode, + CountSumNode, } impl NodeType { @@ -323,6 +330,7 @@ impl NodeType { NodeType::SumNode => 9, NodeType::BigSumNode => 17, NodeType::CountNode => 9, + NodeType::CountSumNode => 17, } } @@ -332,6 +340,7 @@ impl NodeType { NodeType::SumNode => 8, NodeType::BigSumNode => 16, NodeType::CountNode => 8, + NodeType::CountSumNode => 16, } } } diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index 4b8c9f650..5bffb0259 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -289,7 +289,7 @@ impl Link { // sum_len for sum vale key.len() + 44 // 1 + 32 + 2 + 1 + 8 } - AggregateData::BigSum(_) => { + AggregateData::BigSum(_) | AggregateData::CountAndSum(_, _) => { // 1 for key len // key_len for keys // 32 for hash @@ -316,7 +316,7 @@ impl Link { AggregateData::Count(_) | AggregateData::Sum(_) => { tree.key().len() + 44 // 1 + 32 + 2 + 1 + 8 } - AggregateData::BigSum(_) => { + AggregateData::BigSum(_) | AggregateData::CountAndSum(_, _) => { tree.key().len() + 52 // 1 + 32 + 2 + 1 + 16 } }, @@ -376,6 +376,11 @@ impl Encode for Link { out.write_all(&[3])?; out.write_varint(*count_value)?; } + AggregateData::CountAndSum(count_value, sum_value) => { + out.write_all(&[4])?; + out.write_varint(*count_value)?; + out.write_varint(*sum_value)?; + } } Ok(()) @@ -416,7 +421,7 @@ impl Encode for Link { key.len() + 52 // 1 + 32 + 2 + 1 + 16 } AggregateData::Count(count) => { - let encoded_sum_value = count.encode_var_vec(); + let encoded_count_value = count.encode_var_vec(); // 1 for key len // key_len for keys // 32 for hash @@ -425,7 +430,12 @@ impl Encode for Link { // if above is 1, then // 1 for sum len // sum_len for sum vale - key.len() + encoded_sum_value.len() + 36 // 1 + 32 + 2 + 1 + key.len() + encoded_count_value.len() + 36 // 1 + 32 + 2 + 1 + } + AggregateData::CountAndSum(count, sum) => { + let encoded_sum_value = sum.encode_var_vec(); + let encoded_count_value = count.encode_var_vec(); + key.len() + encoded_sum_value.len() + encoded_count_value.len() + 36 } }, Link::Modified { .. } => panic!("No encoding for Link::Modified"), @@ -451,6 +461,11 @@ impl Encode for Link { let encoded_count_value = count_value.encode_var_vec(); tree.key().len() + encoded_count_value.len() + 36 // 1 + 32 + 2 + 1 } + AggregateData::CountAndSum(count, sum) => { + let encoded_sum_value = sum.encode_var_vec(); + let encoded_count_value = count.encode_var_vec(); + tree.key().len() + encoded_sum_value.len() + encoded_count_value.len() + 36 + } }, }) } @@ -518,7 +533,12 @@ impl Decode for Link { let encoded_count: u64 = input.read_varint()?; AggregateData::Count(encoded_count) } - _ => return Err(ed::Error::UnexpectedByte(55)), + 4 => { + let encoded_count: u64 = input.read_varint()?; + let encoded_sum: i64 = input.read_varint()?; + AggregateData::CountAndSum(encoded_count, encoded_sum) + } + byte => return Err(ed::Error::UnexpectedByte(byte)), }; } else { unreachable!() diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 7600246be..8abb7e8c1 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -460,6 +460,7 @@ impl TreeNode { AggregateData::Sum(s) => s.encode_var_vec().len() as u32, AggregateData::BigSum(_) => 16 as u32, AggregateData::Count(c) => c.encode_var_vec().len() as u32, + AggregateData::CountAndSum(c, s) => s.encode_var_vec().len() as u32 + c.encode_var_vec().len() as u32, }, ) }) @@ -501,7 +502,7 @@ impl TreeNode { /// Returns the sum of the root node's child on the given side, if any. If /// there is no child, returns 0. #[inline] - pub fn child_aggregate_data_as_i64(&self, left: bool) -> Result { + pub fn child_aggregate_sum_data_as_i64(&self, left: bool) -> Result { match self.link(left) { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => Ok(0), @@ -509,13 +510,8 @@ impl TreeNode { AggregateData::BigSum(_) => Err(Error::BigSumTreeUnderNormalSumTree( "for aggregate data as i64".to_string(), )), - AggregateData::Count(c) => { - if c > i64::MAX as u64 { - Err(Overflow("count overflow when below sum tree")) - } else { - Ok(c as i64) - } - } + AggregateData::Count(_) => Ok(0), + AggregateData::CountAndSum(_, s) => Ok(s), }, _ => Ok(0), } @@ -524,21 +520,14 @@ impl TreeNode { /// Returns the sum of the root node's child on the given side, if any. If /// there is no child, returns 0. #[inline] - pub fn child_aggregate_data_as_u64(&self, left: bool) -> Result { + pub fn child_aggregate_count_data_as_u64(&self, left: bool) -> Result { match self.link(left) { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => Ok(0), - AggregateData::Sum(s) => { - if s < 0 { - Err(Error::Overflow("negative sum tree under count tree")) - } else { - Ok(s as u64) - } - } - AggregateData::BigSum(_) => Err(Error::BigSumTreeUnderNormalSumTree( - "for aggregate data as u64".to_string(), - )), + AggregateData::Sum(s) => Ok(0), + AggregateData::BigSum(_) => Ok(0), AggregateData::Count(c) => Ok(c), + AggregateData::CountAndSum(c, _) => Ok(c), }, _ => Ok(0), } @@ -547,13 +536,14 @@ impl TreeNode { /// Returns the sum of the root node's child on the given side, if any. If /// there is no child, returns 0. #[inline] - pub fn child_aggregate_data_as_i128(&self, left: bool) -> i128 { + pub fn child_aggregate_sum_data_as_i128(&self, left: bool) -> i128 { match self.link(left) { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => 0, AggregateData::Sum(s) => s as i128, AggregateData::BigSum(s) => s, - AggregateData::Count(c) => c as i128, + AggregateData::Count(_) => 0, + AggregateData::CountAndSum(_, s) => s as i128, }, _ => 0, } @@ -576,8 +566,8 @@ impl TreeNode { match self.inner.kv.feature_type { TreeFeatureType::BasicMerkNode => Ok(AggregateData::NoAggregateData), TreeFeatureType::SummedMerkNode(value) => { - let left = self.child_aggregate_data_as_i64(true)?; - let right = self.child_aggregate_data_as_i64(false)?; + let left = self.child_aggregate_sum_data_as_i64(true)?; + let right = self.child_aggregate_sum_data_as_i64(false)?; value .checked_add(left) .and_then(|a| a.checked_add(right)) @@ -585,19 +575,36 @@ impl TreeNode { .map(AggregateData::Sum) } TreeFeatureType::BigSummedMerkNode(value) => value - .checked_add(self.child_aggregate_data_as_i128(true)) - .and_then(|a| a.checked_add(self.child_aggregate_data_as_i128(false))) + .checked_add(self.child_aggregate_sum_data_as_i128(true)) + .and_then(|a| a.checked_add(self.child_aggregate_sum_data_as_i128(false))) .ok_or(Overflow("big sum is overflowing")) .map(AggregateData::BigSum), TreeFeatureType::CountedMerkNode(value) => { - let left = self.child_aggregate_data_as_u64(true)?; - let right = self.child_aggregate_data_as_u64(false)?; + let left = self.child_aggregate_count_data_as_u64(true)?; + let right = self.child_aggregate_count_data_as_u64(false)?; value .checked_add(left) .and_then(|a| a.checked_add(right)) .ok_or(Overflow("count is overflowing")) .map(AggregateData::Count) } + TreeFeatureType::CountedSummedMerkNode(count_value, sum_value) => { + let left_count = self.child_aggregate_count_data_as_u64(true)?; + let right_count = self.child_aggregate_count_data_as_u64(false)?; + let left_sum = self.child_aggregate_sum_data_as_i64(true)?; + let right_sum = self.child_aggregate_sum_data_as_i64(false)?; + let aggregated_count_value = count_value + .checked_add(left_count) + .and_then(|a| a.checked_add(right_count)) + .ok_or(Overflow("count is overflowing"))?; + + let aggregated_sum_value = sum_value + .checked_add(left_sum) + .and_then(|a| a.checked_add(right_sum)) + .ok_or(Overflow("count is overflowing"))?; + + Ok(AggregateData::CountAndSum(aggregated_count_value, aggregated_sum_value)) + } } } diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 1ecf967e8..cf40320e3 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -17,6 +17,7 @@ use crate::{ merk::{NodeType, TreeType}, TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, }; +use crate::TreeFeatureType::CountedSummedMerkNode; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -30,6 +31,8 @@ pub enum TreeFeatureType { BigSummedMerkNode(i128), /// Counted Merk Tree None CountedMerkNode(u64), + /// Counted and summed Merk Tree None + CountedSummedMerkNode(u64, i64), } impl TreeFeatureType { @@ -39,6 +42,7 @@ impl TreeFeatureType { SummedMerkNode(_) => NodeType::SumNode, BigSummedMerkNode(_) => NodeType::BigSumNode, CountedMerkNode(_) => NodeType::CountNode, + CountedSummedMerkNode(_, _) => NodeType::CountSumNode, } } } @@ -50,6 +54,7 @@ pub enum AggregateData { Sum(i64), BigSum(i128), Count(u64), + CountAndSum(u64, i64) } impl AggregateData { @@ -59,10 +64,11 @@ impl AggregateData { AggregateData::Sum(_) => TreeType::SumTree, AggregateData::BigSum(_) => TreeType::BigSumTree, AggregateData::Count(_) => TreeType::CountTree, + AggregateData::CountAndSum(_, _) => TreeType::CountSumTree, } } - pub fn as_i64(&self) -> i64 { + pub fn as_sum_i64(&self) -> i64 { match self { AggregateData::NoAggregateData => 0, AggregateData::Sum(s) => *s, @@ -82,10 +88,11 @@ impl AggregateData { *c as i64 } } + AggregateData::CountAndSum(_, s) => *s, } } - pub fn as_u64(&self) -> u64 { + pub fn as_count_u64(&self) -> u64 { match self { AggregateData::NoAggregateData => 0, AggregateData::Sum(s) => { @@ -106,6 +113,7 @@ impl AggregateData { } } AggregateData::Count(c) => *c, + AggregateData::CountAndSum(c, _) => *c, } } @@ -115,6 +123,7 @@ impl AggregateData { AggregateData::Sum(s) => *s as i128, AggregateData::BigSum(i) => *i, AggregateData::Count(c) => *c as i128, + AggregateData::CountAndSum(c, _) => *c as i128, } } } @@ -126,6 +135,7 @@ impl From for AggregateData { SummedMerkNode(val) => AggregateData::Sum(val), BigSummedMerkNode(val) => AggregateData::BigSum(val), CountedMerkNode(val) => AggregateData::Count(val), + CountedSummedMerkNode(count, sum) => AggregateData::CountAndSum(count, sum), } } } @@ -134,12 +144,13 @@ impl From for AggregateData { impl TreeFeatureType { #[inline] /// Get length of encoded SummedMerk - pub fn sum_length(&self) -> Option { + pub fn tree_feature_length(&self) -> Option { match self { BasicMerkNode => None, SummedMerkNode(m) => Some(m.encode_var_vec().len() as u32), BigSummedMerkNode(_) => Some(16), CountedMerkNode(m) => Some(m.encode_var_vec().len() as u32), + CountedSummedMerkNode(count, sum) => Some(count.encode_var_vec().len() as u32 + sum.encode_var_vec().len() as u32), } } @@ -151,6 +162,7 @@ impl TreeFeatureType { SummedMerkNode(_sum) => 9, BigSummedMerkNode(_) => 17, CountedMerkNode(_) => 9, + CountedSummedMerkNode(..) => 17, } } } @@ -181,6 +193,12 @@ impl Encode for TreeFeatureType { dest.write_varint(*count)?; Ok(()) } + CountedSummedMerkNode(count, sum) => { + dest.write_all(&[4])?; + dest.write_varint(*count)?; + dest.write_varint(*sum)?; + Ok(()) + } } } @@ -201,6 +219,11 @@ impl Encode for TreeFeatureType { // encoded_sum.len() for the length of the encoded vector Ok(1 + encoded_sum.len()) } + CountedSummedMerkNode(count, sum) => { + let encoded_lengths = count.encode_var_vec().len() + sum.encode_var_vec().len(); + // 1 for the enum type + Ok(1 + encoded_lengths) + } } } } @@ -225,6 +248,11 @@ impl Decode for TreeFeatureType { let encoded_count: u64 = input.read_varint()?; Ok(CountedMerkNode(encoded_count)) } + [4] => { + let encoded_count: u64 = input.read_varint()?; + let encoded_sum: i64 = input.read_varint()?; + Ok(CountedSummedMerkNode(encoded_count, encoded_sum)) + } _ => Err(ed::Error::UnexpectedByte(55)), } } diff --git a/node-grove/src/converter.rs b/node-grove/src/converter.rs index 2aca7b0d9..7ba3850c2 100644 --- a/node-grove/src/converter.rs +++ b/node-grove/src/converter.rs @@ -39,7 +39,8 @@ fn element_to_string(element: Element) -> String { Element::Tree(..) => "tree".to_string(), Element::SumTree(..) => "sum_tree".to_string(), Element::BigSumTree(..) => "big_sum_tree".to_string(), - Element::CountTree(..) => "big_sum_tree".to_string(), + Element::CountTree(..) => "count_tree".to_string(), + Element::CountSumTree(..) => "count_sum_tree".to_string(), } } @@ -96,6 +97,7 @@ pub fn element_to_js_object<'a, C: Context<'a>>( Element::SumTree(..) => nested_vecs_to_js(vec![], cx)?, Element::BigSumTree(..) => nested_vecs_to_js(vec![], cx)?, Element::CountTree(..) => nested_vecs_to_js(vec![], cx)?, + Element::CountSumTree(..) => nested_vecs_to_js(vec![], cx)?, }; js_object.set(cx, "value", js_value)?; From 1ba42f567ac4581f300962aa5e5ef5fae7b9dcd0 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 04:25:15 +0700 Subject: [PATCH 08/30] fixes --- costs/src/lib.rs | 35 +++++++-- .../estimated_costs/average_case_costs.rs | 4 +- grovedb/src/batch/mod.rs | 19 +++-- grovedb/src/element/get.rs | 2 +- grovedb/src/element/helpers.rs | 11 +-- grovedb/src/element/mod.rs | 2 +- grovedb/src/operations/get/mod.rs | 3 +- grovedb/src/operations/get/query.rs | 11 +-- grovedb/src/tests/sum_tree_tests.rs | 71 +++++++++++-------- merk/src/estimated_costs/mod.rs | 3 +- merk/src/merk/committer.rs | 3 +- merk/src/merk/mod.rs | 6 +- merk/src/tree/link.rs | 4 +- merk/src/tree/mod.rs | 11 ++- merk/src/tree/tree_feature_type.rs | 29 +++++--- 15 files changed, 137 insertions(+), 77 deletions(-) diff --git a/costs/src/lib.rs b/costs/src/lib.rs index c3d855307..7e576dc1b 100644 --- a/costs/src/lib.rs +++ b/costs/src/lib.rs @@ -72,9 +72,29 @@ pub type ChildrenSizesWithValue = Option<( Option<(ChildKeyLength, ChildSumLength)>, )>; +/// The tree cost type +pub enum TreeCostType { + /// This is for sum trees and count trees + TreeFeatureUsesVarIntCostAs8Bytes, + /// This is for count sum trees + TreeFeatureUsesTwoVarIntsCostAs16Bytes, + /// This is for big sum trees + TreeFeatureUses16Bytes, +} + +impl TreeCostType { + fn cost_size(&self) -> u32 { + match self { + TreeCostType::TreeFeatureUsesVarIntCostAs8Bytes => 8, + TreeCostType::TreeFeatureUsesTwoVarIntsCostAs16Bytes => 16, + TreeCostType::TreeFeatureUses16Bytes => 16, + } + } +} + /// Children sizes starting with if we are in a sum tree pub type ChildrenSizesWithIsSumTree = Option<( - Option, + Option<(TreeCostType, FeatureSumLength)>, Option<(ChildKeyLength, ChildSumLength)>, Option<(ChildKeyLength, ChildSumLength)>, )>; @@ -199,10 +219,14 @@ impl OperationCost { paid_value_len -= right_child_sum_len; } - if let Some(sum_tree_len) = in_sum_tree { + let sum_tree_node_size = if let Some((tree_cost_type, sum_tree_len)) = in_sum_tree { + let cost_size = tree_cost_type.cost_size(); paid_value_len -= sum_tree_len; - paid_value_len += 8; - } + paid_value_len += cost_size; + cost_size + } else { + 0 + }; // This is the moment we need to add the required space (after removing // children) but before adding the parent to child hook @@ -210,9 +234,6 @@ impl OperationCost { // Now we are the parent to child hook - // we need to add the sum tree node size - let sum_tree_node_size = if in_sum_tree.is_some() { 8 } else { 0 }; - // We need to add the cost of a parent // key_len has a hash length already in it from the key prefix // So we need to remove it and then add a hash length diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 112aac040..cc7cd4521 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -11,8 +11,8 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; #[cfg(feature = "full")] -use grovedb_merk::{ - estimated_costs::average_case_costs::{average_case_merk_propagate, EstimatedLayerInformation}, +use grovedb_merk::estimated_costs::average_case_costs::{ + average_case_merk_propagate, EstimatedLayerInformation, }; use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 1d4316242..e1f8991c9 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -75,8 +75,8 @@ use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, element::{ - MaxReferenceHop, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, - SUM_TREE_COST_SIZE, TREE_COST_SIZE, + MaxReferenceHop, BIG_SUM_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, + SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE, }, operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ @@ -1022,7 +1022,8 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( + | Element::CountTree(..) + | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1143,7 +1144,8 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( + | Element::CountTree(..) + | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1173,7 +1175,8 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( + | Element::CountTree(..) + | Element::CountSumTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1348,7 +1351,8 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) | Element::CountSumTree(..) => { + | Element::CountTree(..) + | Element::CountSumTree(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element @@ -1658,7 +1662,8 @@ where Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) - | Element::CountTree(..) | Element::CountSumTree(..) => { + | Element::CountTree(..) + | Element::CountSumTree(..) => { let tree_type = new_element.tree_type().unwrap(); let tree_cost_size = match tree_type { TreeType::NormalTree => TREE_COST_SIZE, diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 9ff281daa..f9a8dc6e1 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -201,7 +201,7 @@ impl Element { | Some(Element::SumTree(_, _, flags)) | Some(Element::BigSumTree(_, _, flags)) | Some(Element::CountTree(_, _, flags)) - | Some(Element::CountSumTree(.. , flags)) => { + | Some(Element::CountSumTree(.., flags)) => { let tree_cost_size = element.as_ref().unwrap().tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 0acb9d3f4..0ecac7520 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -8,7 +8,7 @@ use grovedb_merk::tree::kv::{ }; use grovedb_merk::{ merk::{NodeType, TreeType}, - TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, }; #[cfg(feature = "full")] use grovedb_merk::{ @@ -20,7 +20,7 @@ use grovedb_merk::{ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; -use grovedb_merk::TreeFeatureType::CountedSummedMerkNode; + use crate::element::{BIG_SUM_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE}; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; @@ -245,7 +245,10 @@ impl Element { TreeType::SumTree => Ok(SummedMerkNode(self.sum_value_or_default())), TreeType::BigSumTree => Ok(BigSummedMerkNode(self.big_sum_value_or_default())), TreeType::CountTree => Ok(CountedMerkNode(self.count_value_or_default())), - TreeType::CountSumTree => Ok(CountedSummedMerkNode(self.count_value_or_default(), self.sum_value_or_default())), + TreeType::CountSumTree => Ok(CountedSummedMerkNode( + self.count_value_or_default(), + self.sum_value_or_default(), + )), } } @@ -416,7 +419,7 @@ impl Element { key_len, value_len, node_type, ) } - Element::CountSumTree(.. , flags) => { + Element::CountSumTree(.., flags) => { let flags_len = flags.map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 5b831a11b..bc3b8bafe 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -24,13 +24,13 @@ mod serialize; use std::fmt; use bincode::{Decode, Encode}; +use grovedb_merk::estimated_costs::SUM_AND_COUNT_LAYER_COST_SIZE; #[cfg(any(feature = "full", feature = "verify"))] use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; #[cfg(feature = "full")] use grovedb_merk::estimated_costs::{ BIG_SUM_LAYER_COST_SIZE, LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, }; -use grovedb_merk::estimated_costs::SUM_AND_COUNT_LAYER_COST_SIZE; #[cfg(feature = "full")] use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index c035147f0..88022d662 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -458,7 +458,8 @@ impl GroveDb { Ok(Element::Tree(..)) | Ok(Element::SumTree(..)) | Ok(Element::BigSumTree(..)) - | Ok(Element::CountTree(..)) | Ok(Element::CountSumTree(..)) => Ok(()).wrap_with_cost(cost), + | Ok(Element::CountTree(..)) + | Ok(Element::CountSumTree(..)) => Ok(()).wrap_with_cost(cost), Ok(_) | Err(Error::PathKeyNotFound(_)) => Err(error_fn()).wrap_with_cost(cost), Err(e) => Err(e).wrap_with_cost(cost), } diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 584e6ce7e..618680dc9 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -445,7 +445,10 @@ where { Ok(QueryItemOrSumReturnType::CountValue(count_value)) } Element::CountSumTree(_, count_value, sum_value, _) => { - Ok(QueryItemOrSumReturnType::CountSumValue(count_value, sum_value)) + Ok(QueryItemOrSumReturnType::CountSumValue( + count_value, + sum_value, + )) } _ => Err(Error::InvalidQuery( "the reference must result in an item", @@ -470,9 +473,9 @@ where { Element::CountTree(_, count_value, _) => { Ok(QueryItemOrSumReturnType::CountValue(count_value)) } - Element::CountSumTree(_, count_value, sum_value, _) => { - Ok(QueryItemOrSumReturnType::CountSumValue(count_value, sum_value)) - } + Element::CountSumTree(_, count_value, sum_value, _) => Ok( + QueryItemOrSumReturnType::CountSumValue(count_value, sum_value), + ), Element::Tree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items, sum items, references and sum \ trees", diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 0c9c19dbc..777fcb453 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -5,9 +5,8 @@ mod tests { use grovedb_merk::{ proofs::Query, tree::{kv::ValueDefinedCostType, AggregateData}, - TreeFeatureType::{BasicMerkNode, SummedMerkNode}, + TreeFeatureType::{BasicMerkNode, BigSummedMerkNode, SummedMerkNode}, }; - use grovedb_merk::TreeFeatureType::BigSummedMerkNode; use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; @@ -324,7 +323,9 @@ mod tests { Some(SummedMerkNode(0)) )); assert_eq!( - merk.aggregate_data().expect("expected to get sum").as_sum_i64(), + merk.aggregate_data() + .expect("expected to get sum") + .as_sum_i64(), 40 ); @@ -1063,8 +1064,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert tree"); + .unwrap() + .expect("should insert tree"); db.insert( [TEST_LEAF, b"big_sum_tree"].as_ref(), b"sum_tree_1", @@ -1073,8 +1074,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert tree"); + .unwrap() + .expect("should insert tree"); db.insert( [TEST_LEAF, b"big_sum_tree"].as_ref(), b"sum_tree_2", @@ -1083,8 +1084,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert tree"); + .unwrap() + .expect("should insert tree"); db.insert( [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), b"item1", @@ -1093,8 +1094,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert item"); + .unwrap() + .expect("should insert item"); db.insert( [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), b"sum_item_1", @@ -1103,8 +1104,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert item"); + .unwrap() + .expect("should insert item"); db.insert( [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), b"sum_item_2", @@ -1113,8 +1114,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert item"); + .unwrap() + .expect("should insert item"); db.insert( [TEST_LEAF, b"big_sum_tree", b"sum_tree_1"].as_ref(), b"ref_1", @@ -1128,8 +1129,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert item"); + .unwrap() + .expect("should insert item"); db.insert( [TEST_LEAF, b"big_sum_tree", b"sum_tree_2"].as_ref(), @@ -1139,14 +1140,17 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert item"); + .unwrap() + .expect("should insert item"); let sum_tree = db .get([TEST_LEAF].as_ref(), b"big_sum_tree", None, grove_version) .unwrap() .expect("should fetch tree"); - assert_eq!(sum_tree.big_sum_value_or_default(), (SumValue::MAX - 10) as i128 + (SumValue::MAX - 50) as i128); + assert_eq!( + sum_tree.big_sum_value_or_default(), + (SumValue::MAX - 10) as i128 + (SumValue::MAX - 50) as i128 + ); db.insert( [TEST_LEAF, b"big_sum_tree"].as_ref(), @@ -1156,14 +1160,19 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("should insert item"); + .unwrap() + .expect("should insert item"); let sum_tree = db .get([TEST_LEAF].as_ref(), b"big_sum_tree", None, grove_version) .unwrap() .expect("should fetch tree"); - assert_eq!(sum_tree.big_sum_value_or_default(), (SumValue::MAX - 10) as i128 + (SumValue::MAX - 50) as i128 + (SumValue::MAX - 70) as i128 ); + assert_eq!( + sum_tree.big_sum_value_or_default(), + (SumValue::MAX - 10) as i128 + + (SumValue::MAX - 50) as i128 + + (SumValue::MAX - 70) as i128 + ); let batch = StorageBatch::new(); @@ -1202,10 +1211,11 @@ mod tests { b"sum_tree_1", true, Some(&Element::value_defined_cost_for_serialized_value), - grove_version + grove_version, ) .unwrap() - .expect("node should exist").expect("expected feature type"); + .expect("node should exist") + .expect("expected feature type"); assert_eq!( feature_type, BigSummedMerkNode((SumValue::MAX - 10) as i128) @@ -1216,10 +1226,11 @@ mod tests { b"sum_item_4", true, Some(&Element::value_defined_cost_for_serialized_value), - grove_version + grove_version, ) .unwrap() - .expect("node should exist").expect("expected feature type"); + .expect("node should exist") + .expect("expected feature type"); assert_eq!( feature_type, BigSummedMerkNode((SumValue::MAX - 70) as i128) @@ -1570,11 +1581,11 @@ mod tests { ) .unwrap() .expect("node should exist"), - Some(SummedMerkNode(10)) + Some(BigSummedMerkNode(10)) ); assert_eq!( sum_tree.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(20) + AggregateData::BigSum(20) ); // Test propagation @@ -1652,7 +1663,7 @@ mod tests { .expect("should open tree"); assert_eq!( sum_tree.aggregate_data().expect("expected to get sum"), - AggregateData::Sum(41) + AggregateData::BigSum(41) ); } } diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index eb2eaa404..5ad58c0a5 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -43,7 +43,8 @@ pub const SUM_LAYER_COST_SIZE: u32 = LAYER_COST_SIZE + SUM_VALUE_EXTRA_COST; #[cfg(feature = "full")] /// The cost of a summed subtree layer /// This is the layer size + 9 for the encoded value -pub const SUM_AND_COUNT_LAYER_COST_SIZE: u32 = LAYER_COST_SIZE + SUM_VALUE_EXTRA_COST + COUNT_VALUE_EXTRA_COST; +pub const SUM_AND_COUNT_LAYER_COST_SIZE: u32 = + LAYER_COST_SIZE + SUM_VALUE_EXTRA_COST + COUNT_VALUE_EXTRA_COST; #[cfg(feature = "full")] /// The cost of a summed subtree layer diff --git a/merk/src/merk/committer.rs b/merk/src/merk/committer.rs index c260c3556..49e4fbdc2 100644 --- a/merk/src/merk/committer.rs +++ b/merk/src/merk/committer.rs @@ -44,7 +44,8 @@ impl Commit for MerkCommitter { let right_child_sizes = tree.child_ref_and_sum_size(false); self.batch.push(( tree.key().to_vec(), - tree.feature_type().tree_feature_length(), + tree.feature_type() + .tree_feature_specialized_type_and_length(), Some((buf, left_child_sizes, right_child_sizes)), storage_costs, )); diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index e4abe7508..bcf537802 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -52,7 +52,7 @@ use committer::MerkCommitter; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_default, cost_return_on_error_no_add, storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithValue, CostContext, - CostResult, CostsExt, FeatureSumLength, OperationCost, + CostResult, CostsExt, FeatureSumLength, OperationCost, TreeCostType, }; use grovedb_storage::{self, Batch, RawIterator, StorageContext}; use grovedb_version::version::GroveVersion; @@ -107,7 +107,7 @@ impl KeyUpdates { /// Type alias for simple function signature pub type BatchValue = ( Vec, - Option, + Option<(TreeCostType, FeatureSumLength)>, ChildrenSizesWithValue, KeyValueStorageCost, ); @@ -367,7 +367,7 @@ impl fmt::Debug for Merk { pub type UseTreeMutResult = CostResult< Vec<( Vec, - Option, + Option<(TreeCostType, FeatureSumLength)>, ChildrenSizesWithValue, KeyValueStorageCost, )>, diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index 5bffb0259..3e826e38d 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -289,7 +289,7 @@ impl Link { // sum_len for sum vale key.len() + 44 // 1 + 32 + 2 + 1 + 8 } - AggregateData::BigSum(_) | AggregateData::CountAndSum(_, _) => { + AggregateData::BigSum(_) | AggregateData::CountAndSum(..) => { // 1 for key len // key_len for keys // 32 for hash @@ -316,7 +316,7 @@ impl Link { AggregateData::Count(_) | AggregateData::Sum(_) => { tree.key().len() + 44 // 1 + 32 + 2 + 1 + 8 } - AggregateData::BigSum(_) | AggregateData::CountAndSum(_, _) => { + AggregateData::BigSum(_) | AggregateData::CountAndSum(..) => { tree.key().len() + 52 // 1 + 32 + 2 + 1 + 16 } }, diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 8abb7e8c1..8a10f82b0 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -460,7 +460,9 @@ impl TreeNode { AggregateData::Sum(s) => s.encode_var_vec().len() as u32, AggregateData::BigSum(_) => 16 as u32, AggregateData::Count(c) => c.encode_var_vec().len() as u32, - AggregateData::CountAndSum(c, s) => s.encode_var_vec().len() as u32 + c.encode_var_vec().len() as u32, + AggregateData::CountAndSum(c, s) => { + s.encode_var_vec().len() as u32 + c.encode_var_vec().len() as u32 + } }, ) }) @@ -525,7 +527,7 @@ impl TreeNode { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => Ok(0), AggregateData::Sum(s) => Ok(0), - AggregateData::BigSum(_) => Ok(0), + AggregateData::BigSum(_) => Ok(0), AggregateData::Count(c) => Ok(c), AggregateData::CountAndSum(c, _) => Ok(c), }, @@ -603,7 +605,10 @@ impl TreeNode { .and_then(|a| a.checked_add(right_sum)) .ok_or(Overflow("count is overflowing"))?; - Ok(AggregateData::CountAndSum(aggregated_count_value, aggregated_sum_value)) + Ok(AggregateData::CountAndSum( + aggregated_count_value, + aggregated_sum_value, + )) } } } diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index cf40320e3..f83199d7e 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -8,6 +8,7 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use ed::Terminated; #[cfg(any(feature = "full", feature = "verify"))] use ed::{Decode, Encode}; +use grovedb_costs::TreeCostType; #[cfg(any(feature = "full", feature = "verify"))] use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; @@ -15,9 +16,8 @@ use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; use crate::{ merk::{NodeType, TreeType}, - TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, }; -use crate::TreeFeatureType::CountedSummedMerkNode; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -42,7 +42,7 @@ impl TreeFeatureType { SummedMerkNode(_) => NodeType::SumNode, BigSummedMerkNode(_) => NodeType::BigSumNode, CountedMerkNode(_) => NodeType::CountNode, - CountedSummedMerkNode(_, _) => NodeType::CountSumNode, + CountedSummedMerkNode(..) => NodeType::CountSumNode, } } } @@ -54,7 +54,7 @@ pub enum AggregateData { Sum(i64), BigSum(i128), Count(u64), - CountAndSum(u64, i64) + CountAndSum(u64, i64), } impl AggregateData { @@ -64,7 +64,7 @@ impl AggregateData { AggregateData::Sum(_) => TreeType::SumTree, AggregateData::BigSum(_) => TreeType::BigSumTree, AggregateData::Count(_) => TreeType::CountTree, - AggregateData::CountAndSum(_, _) => TreeType::CountSumTree, + AggregateData::CountAndSum(..) => TreeType::CountSumTree, } } @@ -144,13 +144,22 @@ impl From for AggregateData { impl TreeFeatureType { #[inline] /// Get length of encoded SummedMerk - pub fn tree_feature_length(&self) -> Option { + pub fn tree_feature_specialized_type_and_length(&self) -> Option<(TreeCostType, u32)> { match self { BasicMerkNode => None, - SummedMerkNode(m) => Some(m.encode_var_vec().len() as u32), - BigSummedMerkNode(_) => Some(16), - CountedMerkNode(m) => Some(m.encode_var_vec().len() as u32), - CountedSummedMerkNode(count, sum) => Some(count.encode_var_vec().len() as u32 + sum.encode_var_vec().len() as u32), + SummedMerkNode(m) => Some(( + TreeCostType::TreeFeatureUsesVarIntCostAs8Bytes, + m.encode_var_vec().len() as u32, + )), + BigSummedMerkNode(_) => Some((TreeCostType::TreeFeatureUses16Bytes, 16)), + CountedMerkNode(m) => Some(( + TreeCostType::TreeFeatureUsesVarIntCostAs8Bytes, + m.encode_var_vec().len() as u32, + )), + CountedSummedMerkNode(count, sum) => Some(( + TreeCostType::TreeFeatureUsesTwoVarIntsCostAs16Bytes, + count.encode_var_vec().len() as u32 + sum.encode_var_vec().len() as u32, + )), } } From 7d91f007e0204ccdd34af08ac5f7dc4f915bbc67 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 05:29:37 +0700 Subject: [PATCH 09/30] more work --- grovedb/src/batch/mod.rs | 11 + grovedb/src/element/helpers.rs | 23 +- grovedb/src/lib.rs | 53 ++- grovedb/src/tests/count_sum_tree_tests.rs | 556 ++++++++++++++++++++++ grovedb/src/tests/mod.rs | 1 + merk/src/tree/tree_feature_type.rs | 34 +- 6 files changed, 631 insertions(+), 47 deletions(-) create mode 100644 grovedb/src/tests/count_sum_tree_tests.rs diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index e1f8991c9..d340cfb84 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1900,6 +1900,17 @@ impl GroveDb { aggregate_data, } .into(); + } else if let Element::CountSumTree(.., flags) = + element + { + *mutable_occupied_entry = + GroveOp::InsertTreeWithRootHash { + hash: root_hash, + root_key: calculated_root_key, + flags: flags.clone(), + aggregate_data, + } + .into(); } else { return Err(Error::InvalidBatchOperation( "insertion of element under a non tree", diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 0ecac7520..5a02f9c3d 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -55,6 +55,18 @@ impl Element { } } + #[cfg(any(feature = "full", feature = "verify"))] + /// Decoded the integer value in the CountTree element type, returns 1 for + /// everything else + pub fn count_sum_value_or_default(&self) -> (u64, i64) { + match self { + Element::SumItem(sum_value, _) | Element::SumTree(_, sum_value, _) => (1, *sum_value), + Element::CountTree(_, count_value, _) => (*count_value, 0), + Element::CountSumTree(_, count_value, sum_value, _) => (*count_value, *sum_value), + _ => (1, 0), + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Decoded the integer value in the SumItem element type, returns 0 for /// everything else @@ -146,6 +158,7 @@ impl Element { Element::SumTree(root_key, ..) => Some((root_key, TreeType::SumTree)), Element::BigSumTree(root_key, ..) => Some((root_key, TreeType::BigSumTree)), Element::CountTree(root_key, ..) => Some((root_key, TreeType::CountTree)), + Element::CountSumTree(root_key, ..) => Some((root_key, TreeType::CountSumTree)), _ => None, } } @@ -159,6 +172,7 @@ impl Element { Element::SumTree(root_key, ..) => Some((root_key, TreeType::SumTree)), Element::BigSumTree(root_key, ..) => Some((root_key, TreeType::BigSumTree)), Element::CountTree(root_key, ..) => Some((root_key, TreeType::CountTree)), + Element::CountSumTree(root_key, ..) => Some((root_key, TreeType::CountSumTree)), _ => None, } } @@ -171,6 +185,7 @@ impl Element { Element::SumTree(_, _, flags) => Some((flags, TreeType::SumTree)), Element::BigSumTree(_, _, flags) => Some((flags, TreeType::BigSumTree)), Element::CountTree(_, _, flags) => Some((flags, TreeType::CountTree)), + Element::CountSumTree(.., flags) => Some((flags, TreeType::CountSumTree)), _ => None, } } @@ -245,10 +260,10 @@ impl Element { TreeType::SumTree => Ok(SummedMerkNode(self.sum_value_or_default())), TreeType::BigSumTree => Ok(BigSummedMerkNode(self.big_sum_value_or_default())), TreeType::CountTree => Ok(CountedMerkNode(self.count_value_or_default())), - TreeType::CountSumTree => Ok(CountedSummedMerkNode( - self.count_value_or_default(), - self.sum_value_or_default(), - )), + TreeType::CountSumTree => { + let v = self.count_sum_value_or_default(); + Ok(CountedSummedMerkNode(v.0, v.1)) + } } } diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a757dade4..0bc1eb31a 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -421,12 +421,7 @@ impl GroveDb { )) }) .unwrap()?; - let tree_type = element.tree_type(); - if let Element::Tree(root_key, _) - | Element::SumTree(root_key, ..) - | Element::BigSumTree(root_key, ..) = element - { - let tree_type = tree_type.expect("expected tree type"); + if let Some((root_key, tree_type)) = element.root_key_and_tree_type_owned() { Ok(( Merk::open_layered_with_root_key( storage, @@ -498,13 +493,7 @@ impl GroveDb { } ) ); - let tree_type = element.tree_type(); - if let Element::Tree(root_key, _) - | Element::SumTree(root_key, ..) - | Element::BigSumTree(root_key, ..) - | Element::CountTree(root_key, ..) = element - { - let tree_type = tree_type.expect("expected tree type"); + if let Some((root_key, tree_type)) = element.root_key_and_tree_type_owned() { Merk::open_layered_with_root_key( storage, root_key, @@ -823,7 +812,7 @@ impl GroveDb { } else if let Element::BigSumTree(.., flag) = element { let tree = Element::new_big_sum_tree_with_flags_and_sum_value( maybe_root_key, - aggregate_data.as_i128(), + aggregate_data.as_summed_i128(), flag, ); tree.insert_subtree( @@ -846,6 +835,20 @@ impl GroveDb { None, grove_version, ) + } else if let Element::CountSumTree(.., flag) = element { + let tree = Element::new_count_sum_tree_with_flags_and_sum_and_count_value( + maybe_root_key, + aggregate_data.as_count_u64(), + aggregate_data.as_sum_i64(), + flag, + ); + tree.insert_subtree( + parent_tree, + key.as_ref(), + root_tree_hash, + None, + grove_version, + ) } else { Err(Error::InvalidPath( "can only propagate on tree items".to_owned(), @@ -910,7 +913,7 @@ impl GroveDb { } else if let Element::BigSumTree(.., flag) = element { let tree = Element::new_big_sum_tree_with_flags_and_sum_value( maybe_root_key, - aggregate_data.as_i128(), + aggregate_data.as_summed_i128(), flag, ); let merk_feature_type = cost_return_on_error!( @@ -945,6 +948,26 @@ impl GroveDb { merk_feature_type, grove_version, ) + } else if let Element::CountSumTree(.., flag) = element { + let tree = Element::new_count_sum_tree_with_flags_and_sum_and_count_value( + maybe_root_key, + aggregate_data.as_count_u64(), + aggregate_data.as_sum_i64(), + flag, + ); + let merk_feature_type = cost_return_on_error!( + &mut cost, + tree.get_feature_type(parent_tree.tree_type) + .wrap_with_cost(OperationCost::default()) + ); + tree.insert_subtree_into_batch_operations( + key, + root_tree_hash, + true, + batch_operations, + merk_feature_type, + grove_version, + ) } else { Err(Error::InvalidPath( "can only propagate on tree items".to_owned(), diff --git a/grovedb/src/tests/count_sum_tree_tests.rs b/grovedb/src/tests/count_sum_tree_tests.rs new file mode 100644 index 000000000..f171aee0d --- /dev/null +++ b/grovedb/src/tests/count_sum_tree_tests.rs @@ -0,0 +1,556 @@ +//! Count sum tree tests + +#[cfg(test)] +mod count_sum_tree_tests { + use grovedb_merk::{ + tree::{kv::ValueDefinedCostType, AggregateData}, + TreeFeatureType, + }; + use grovedb_storage::StorageBatch; + use grovedb_version::version::GroveVersion; + + use crate::{ + batch::QualifiedGroveDbOp, + tests::{make_test_grovedb, TEST_LEAF}, + Element, + }; + + #[test] + fn test_count_sum_tree_behaves_like_regular_tree() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Insert a CountSumTree + db.insert( + [TEST_LEAF].as_ref(), + b"count_sum_key", + Element::new_count_sum_tree(None), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert CountSumTree"); + + // Fetch the CountSumTree + let count_sum_tree = db + .get([TEST_LEAF].as_ref(), b"count_sum_key", None, grove_version) + .unwrap() + .expect("should get CountSumTree"); + assert!(matches!(count_sum_tree, Element::CountSumTree(..))); + + // Insert items into the CountSumTree + db.insert( + [TEST_LEAF, b"count_sum_key"].as_ref(), + b"item1", + Element::new_item(vec![1]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item1"); + + db.insert( + [TEST_LEAF, b"count_sum_key"].as_ref(), + b"item2", + Element::new_sum_item(3), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item2"); + + db.insert( + [TEST_LEAF, b"count_sum_key"].as_ref(), + b"item3", + Element::new_sum_item(5), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item3"); + + // Test proper item retrieval + let item1 = db + .get( + [TEST_LEAF, b"count_sum_key"].as_ref(), + b"item1", + None, + grove_version, + ) + .unwrap() + .expect("should get item1"); + assert_eq!(item1, Element::new_item(vec![1])); + + let item2 = db + .get( + [TEST_LEAF, b"count_sum_key"].as_ref(), + b"item2", + None, + grove_version, + ) + .unwrap() + .expect("should get item2"); + assert_eq!(item2, Element::new_sum_item(3)); + + let item3 = db + .get( + [TEST_LEAF, b"count_sum_key"].as_ref(), + b"item3", + None, + grove_version, + ) + .unwrap() + .expect("should get item3"); + assert_eq!(item3, Element::new_sum_item(5)); + + // Test aggregate data (count and sum) + let batch = StorageBatch::new(); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"count_sum_key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open CountSumTree"); + + let aggregate_data = merk + .aggregate_data() + .expect("expected to get aggregate data"); + + // Assuming AggregateData::CountAndSum is implemented + assert_eq!(aggregate_data, AggregateData::CountAndSum(3, 8)); // 3 items: 1, 3, 5 + } + + #[test] + fn test_count_sum_tree_item_behaves_like_regular_item() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Insert a CountSumTree with flags + db.insert( + [TEST_LEAF].as_ref(), + b"count_sum_key2", + Element::new_count_sum_tree_with_flags(None, None), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert CountSumTree with flags"); + + // Insert count and sum items + db.insert( + [TEST_LEAF, b"count_sum_key2"].as_ref(), + b"count_item", + Element::new_item(vec![2]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert count_item"); + + db.insert( + [TEST_LEAF, b"count_sum_key2"].as_ref(), + b"sum_item", + Element::new_sum_item(4), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert sum_item"); + + // Test aggregate data + let batch = StorageBatch::new(); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"count_sum_key2"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open CountSumTree with flags"); + + let aggregate_data = merk + .aggregate_data() + .expect("expected to get aggregate data"); + + assert_eq!(aggregate_data, AggregateData::CountAndSum(2, 4)); + } + + #[test] + fn test_homogenous_node_type_in_count_sum_trees_and_regular_trees() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Insert a CountSumTree with initial sum and count values + db.insert( + [TEST_LEAF].as_ref(), + b"count_sum_key3", + Element::new_count_sum_tree_with_flags_and_sum_and_count_value(None, 0, 0, None), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert CountSumTree with sum and count values"); + + // Add count and sum items + db.insert( + [TEST_LEAF, b"count_sum_key3"].as_ref(), + b"item1", + Element::new_item(vec![10]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item1"); + + db.insert( + [TEST_LEAF, b"count_sum_key3"].as_ref(), + b"item2", + Element::new_sum_item(20), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item2"); + + // Add regular items + db.insert( + [TEST_LEAF, b"count_sum_key3"].as_ref(), + b"item3", + Element::new_item(vec![30]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item3"); + + db.insert( + [TEST_LEAF, b"count_sum_key3"].as_ref(), + b"item4", + Element::new_item(vec![40]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item4"); + + // Open merk and check all elements in it + let batch = StorageBatch::new(); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"count_sum_key3"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open CountSumTree"); + + // Verify feature types + let feature_type_item1 = merk + .get_feature_type( + b"item1", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + assert_eq!( + feature_type_item1, + TreeFeatureType::CountedSummedMerkNode(1, 0) + ); + + let feature_type_item2 = merk + .get_feature_type( + b"item2", + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + assert_eq!( + feature_type_item2, + TreeFeatureType::CountedSummedMerkNode(1, 20) + ); + + let feature_type_item3 = merk + .get_feature_type( + b"item3", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + assert_eq!( + feature_type_item3, + TreeFeatureType::CountedSummedMerkNode(1, 0) + ); + + let feature_type_item4 = merk + .get_feature_type( + b"item4", + true, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap() + .expect("node should exist") + .expect("expected feature type"); + assert_eq!( + feature_type_item4, + TreeFeatureType::CountedSummedMerkNode(1, 0) + ); + + // Verify aggregate data + let aggregate_data = merk + .aggregate_data() + .expect("expected to get aggregate data"); + assert_eq!(aggregate_data, AggregateData::CountAndSum(4, 20)); // 2 count, 10 + 20 sum + } + + #[test] + fn test_count_sum_tree_feature() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Insert a regular tree + db.insert( + [TEST_LEAF].as_ref(), + b"regular_key", + Element::new_tree(None), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert regular tree"); + + let batch = StorageBatch::new(); + + // Aggregate data should be None for regular tree + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"regular_key"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open regular tree"); + assert_eq!( + merk.aggregate_data() + .expect("expected to get aggregate data"), + AggregateData::NoAggregateData + ); + + // Insert a CountSumTree + db.insert( + [TEST_LEAF].as_ref(), + b"count_sum_key4", + Element::new_count_sum_tree_with_flags_and_sum_and_count_value(None, 0, 0, None), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert CountSumTree"); + + let count_sum_tree = db + .get([TEST_LEAF].as_ref(), b"count_sum_key4", None, grove_version) + .unwrap() + .expect("should retrieve CountSumTree"); + assert!(matches!(count_sum_tree, Element::CountSumTree(..))); + // Note: Directly accessing count_sum_value_or_default is not shown in original + // code. Assuming you have a method like this to extract count and sum + // from the Element. If not, rely on aggregate_data as below. + + // Add count and sum items + db.insert( + [TEST_LEAF, b"count_sum_key4"].as_ref(), + b"count_item1", + Element::new_item(vec![1]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert count_item1"); + + db.insert( + [TEST_LEAF, b"count_sum_key4"].as_ref(), + b"sum_item1", + Element::new_sum_item(5), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert sum_item1"); + + // Verify aggregate data + let batch = StorageBatch::new(); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"count_sum_key4"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open CountSumTree"); + + let aggregate_data = merk + .aggregate_data() + .expect("expected to get aggregate data"); + assert_eq!(aggregate_data, AggregateData::CountAndSum(2, 5)); + } + + #[test] + fn test_count_sum_tree_with_batches() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Prepare a batch of operations + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"count_sum_key6".to_vec(), + Element::new_count_sum_tree(None), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"count_sum_key6".to_vec()], + b"a".to_vec(), + Element::new_item(vec![10]), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec(), b"count_sum_key6".to_vec()], + b"b".to_vec(), + Element::new_sum_item(20), + ), + ]; + + // Apply the batch + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("should apply batch"); + + // Open the CountSumTree and verify aggregate data + let batch = StorageBatch::new(); + let merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"count_sum_key6"].as_ref().into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open CountSumTree"); + + let aggregate_data = merk + .aggregate_data() + .expect("expected to get aggregate data"); + assert_eq!(aggregate_data, AggregateData::CountAndSum(2, 20)); + } + + #[test] + fn test_count_sum_tree_propagation() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + // Insert a parent CountSumTree + db.insert( + [TEST_LEAF].as_ref(), + b"parent_count_sum", + Element::new_count_sum_tree(None), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert parent CountSumTree"); + + // Insert a child CountSumTree within the parent + db.insert( + [TEST_LEAF, b"parent_count_sum"].as_ref(), + b"child_count_sum", + Element::new_count_sum_tree(None), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert child CountSumTree"); + + // Insert items into the child CountSumTree + db.insert( + [TEST_LEAF, b"parent_count_sum", b"child_count_sum"].as_ref(), + b"item1", + Element::new_item(vec![5]), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item1 into child"); + + db.insert( + [TEST_LEAF, b"parent_count_sum", b"child_count_sum"].as_ref(), + b"item2", + Element::new_sum_item(15), + None, + None, + grove_version, + ) + .unwrap() + .expect("should insert item2 into child"); + + // Verify aggregate data of child + let batch = StorageBatch::new(); + let child_merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"parent_count_sum", b"child_count_sum"] + .as_ref() + .into(), + Some(&batch), + grove_version, + ) + .unwrap() + .expect("should open child CountSumTree"); + + let child_aggregate = child_merk + .aggregate_data() + .expect("expected to get aggregate data"); + assert_eq!(child_aggregate, AggregateData::CountAndSum(2, 15)); + + // Verify aggregate data of parent + let parent_batch = StorageBatch::new(); + let parent_merk = db + .open_non_transactional_merk_at_path( + [TEST_LEAF, b"parent_count_sum"].as_ref().into(), + Some(&parent_batch), + grove_version, + ) + .unwrap() + .expect("should open parent CountSumTree"); + + let parent_aggregate = parent_merk + .aggregate_data() + .expect("expected to get aggregate data"); + assert_eq!(parent_aggregate, AggregateData::CountAndSum(2, 15)); + } +} diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index bd6561ff1..a7f01eb7b 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -6,6 +6,7 @@ mod query_tests; mod sum_tree_tests; +mod count_sum_tree_tests; mod count_tree_tests; mod tree_hashes_tests; diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index f83199d7e..8c9577bcc 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -80,14 +80,7 @@ impl AggregateData { *i as i64 } } - AggregateData::Count(c) => { - let max = i64::MAX as u64; - if *c > max { - i64::MAX - } else { - *c as i64 - } - } + AggregateData::Count(_) => 0, AggregateData::CountAndSum(_, s) => *s, } } @@ -95,35 +88,20 @@ impl AggregateData { pub fn as_count_u64(&self) -> u64 { match self { AggregateData::NoAggregateData => 0, - AggregateData::Sum(s) => { - if *s < 0 { - 0 - } else { - *s as u64 - } - } - AggregateData::BigSum(i) => { - let max = u64::MAX as i128; - if *i > max { - u64::MAX - } else if *i < 0 { - 0 - } else { - *i as u64 - } - } + AggregateData::Sum(_) => 0, + AggregateData::BigSum(_) => 0, AggregateData::Count(c) => *c, AggregateData::CountAndSum(c, _) => *c, } } - pub fn as_i128(&self) -> i128 { + pub fn as_summed_i128(&self) -> i128 { match self { AggregateData::NoAggregateData => 0, AggregateData::Sum(s) => *s as i128, AggregateData::BigSum(i) => *i, - AggregateData::Count(c) => *c as i128, - AggregateData::CountAndSum(c, _) => *c as i128, + AggregateData::Count(_) => 0, + AggregateData::CountAndSum(_, s) => *s as i128, } } } From e2d68fb7623d0ab393b194228bbe71a0c7949e35 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 05:31:15 +0700 Subject: [PATCH 10/30] cleanup --- merk/src/proofs/tree.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/merk/src/proofs/tree.rs b/merk/src/proofs/tree.rs index 6a1a93e53..4b1276007 100644 --- a/merk/src/proofs/tree.rs +++ b/merk/src/proofs/tree.rs @@ -18,12 +18,8 @@ use crate::{error::Error, tree::CryptoHash}; #[cfg(feature = "full")] use crate::{ proofs::chunk::chunk::{LEFT, RIGHT}, - Link, - TreeFeatureType::SummedMerkNode, -}; -use crate::{ tree::AggregateData, - TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, + Link, }; #[cfg(any(feature = "full", feature = "verify"))] @@ -526,6 +522,7 @@ where #[cfg(test)] mod test { use super::{super::*, Tree as ProofTree, *}; + use crate::TreeFeatureType::SummedMerkNode; fn make_7_node_prooftree() -> ProofTree { let make_node = |i| -> super::super::tree::Tree { Node::KV(vec![i], vec![]).into() }; From 85aeb8cab5feead4126ebb2df2fab804efd0a89c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 05:54:14 +0700 Subject: [PATCH 11/30] cleanup --- .../estimated_costs/average_case_costs.rs | 4 +- .../batch/estimated_costs/worst_case_costs.rs | 2 +- .../batch/just_in_time_reference_update.rs | 2 +- grovedb/src/batch/mod.rs | 2 +- .../src/batch/single_deletion_cost_tests.rs | 2 +- .../single_sum_item_deletion_cost_tests.rs | 2 +- grovedb/src/element/delete.rs | 2 +- grovedb/src/element/get.rs | 2 +- grovedb/src/element/helpers.rs | 2 +- grovedb/src/element/mod.rs | 2 +- grovedb/src/element/query.rs | 2 +- .../src/estimated_costs/average_case_costs.rs | 7 +- .../src/estimated_costs/worst_case_costs.rs | 4 +- grovedb/src/lib.rs | 2 +- grovedb/src/operations/delete/average_case.rs | 2 +- .../src/operations/delete/delete_up_tree.rs | 2 +- grovedb/src/operations/delete/mod.rs | 2 +- grovedb/src/operations/delete/worst_case.rs | 2 +- grovedb/src/operations/get/average_case.rs | 2 +- grovedb/src/operations/get/worst_case.rs | 2 +- grovedb/src/operations/is_empty_tree.rs | 2 +- grovedb/src/replication.rs | 4 +- grovedb/src/replication/state_sync_session.rs | 2 +- .../src/estimated_costs/average_case_costs.rs | 2 +- merk/src/estimated_costs/mod.rs | 2 +- merk/src/lib.rs | 3 +- merk/src/merk/mod.rs | 82 ++----------------- merk/src/merk/open.rs | 5 +- merk/src/merk/restore.rs | 4 +- merk/src/merk/source.rs | 2 +- merk/src/merk/tree_type.rs | 72 ++++++++++++++++ merk/src/test_utils/mod.rs | 2 +- merk/src/test_utils/temp_merk.rs | 2 +- merk/src/tree/tree_feature_type.rs | 2 +- 34 files changed, 121 insertions(+), 114 deletions(-) create mode 100644 merk/src/merk/tree_type.rs diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index cc7cd4521..52b42fc65 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -14,7 +14,7 @@ use grovedb_costs::{ use grovedb_merk::estimated_costs::average_case_costs::{ average_case_merk_propagate, EstimatedLayerInformation, }; -use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; +use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; @@ -303,7 +303,7 @@ mod tests { EstimatedLayerSizes::{AllItems, AllSubtrees}, EstimatedSumTrees::{NoSumTrees, SomeSumTrees}, }, - merk::TreeType, + merk::tree_type::TreeType, }; use grovedb_version::version::GroveVersion; diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index b22368731..aecfbadc2 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -14,7 +14,7 @@ use grovedb_costs::{ use grovedb_merk::estimated_costs::worst_case_costs::{ worst_case_merk_propagate, WorstCaseLayerInformation, }; -use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; +use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index e0fd5f94c..26242b400 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -9,7 +9,7 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ - merk::TreeType, + merk::tree_type::TreeType, tree::{kv::KV, value_hash, TreeNode}, CryptoHash, Merk, }; diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index d340cfb84..64027014c 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -48,7 +48,7 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ - merk::TreeType, + merk::tree_type::TreeType, tree::{ kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, value_hash, AggregateData, NULL_HASH, diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index ee5c6902b..e2440377a 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -7,7 +7,7 @@ mod tests { Identifier, StorageRemovalPerEpochByIdentifier, StorageRemovedBytes::SectionedStorageRemoval, }; - use grovedb_merk::merk::TreeType; + use grovedb_merk::merk::tree_type::TreeType; use grovedb_version::version::GroveVersion; use intmap::IntMap; diff --git a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs index 69391631a..73a48f39c 100644 --- a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs @@ -2,7 +2,7 @@ #[cfg(feature = "full")] mod tests { - use grovedb_merk::merk::TreeType; + use grovedb_merk::merk::tree_type::TreeType; use grovedb_version::version::GroveVersion; use crate::{ diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index 41feb6400..596577c63 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -5,7 +5,7 @@ use grovedb_costs::OperationCost; #[cfg(feature = "full")] use grovedb_costs::{storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt}; -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::tree_type::TreeType; #[cfg(feature = "full")] use grovedb_merk::{BatchEntry, Error as MerkError, Merk, MerkOptions, Op}; #[cfg(feature = "full")] diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index f9a8dc6e1..66a9b3ca7 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -365,7 +365,7 @@ impl Element { #[cfg(feature = "full")] #[cfg(test)] mod tests { - use grovedb_merk::merk::TreeType; + use grovedb_merk::merk::tree_type::TreeType; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage, StorageBatch}; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 5a02f9c3d..09775d483 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -7,7 +7,7 @@ use grovedb_merk::tree::kv::{ ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, }; use grovedb_merk::{ - merk::{NodeType, TreeType}, + merk::{tree_type::TreeType, NodeType}, TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, }; #[cfg(feature = "full")] diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index bc3b8bafe..2e0e8af50 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -32,7 +32,7 @@ use grovedb_merk::estimated_costs::{ BIG_SUM_LAYER_COST_SIZE, LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, }; #[cfg(feature = "full")] -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::tree_type::TreeType; #[cfg(feature = "full")] use grovedb_visualize::visualize_to_vec; diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 0df089719..f14d3c735 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -9,7 +9,7 @@ use grovedb_costs::{ OperationCost, }; #[cfg(feature = "full")] -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::tree_type::TreeType; #[cfg(feature = "full")] use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(feature = "full")] diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index ed07857ee..bb0824cb4 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -12,7 +12,7 @@ use grovedb_merk::{ add_average_case_merk_replace_layered, EstimatedLayerInformation, }, }, - merk::TreeType, + merk::tree_type::TreeType, tree::TreeNode, HASH_LENGTH, }; @@ -615,8 +615,9 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ - estimated_costs::average_case_costs::add_average_case_get_merk_node, merk::TreeType, - test_utils::make_batch_seq, tree::kv::ValueDefinedCostType, Merk, + estimated_costs::average_case_costs::add_average_case_get_merk_node, + merk::tree_type::TreeType, test_utils::make_batch_seq, tree::kv::ValueDefinedCostType, + Merk, }; use grovedb_storage::{ rocksdb_storage::RocksDbStorage, worst_case_costs::WorstKeyLength, Storage, StorageBatch, diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index 54cabcc42..681169512 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -14,7 +14,7 @@ use grovedb_merk::{ MERK_BIGGEST_VALUE_SIZE, }, }, - merk::TreeType, + merk::tree_type::TreeType, tree::TreeNode, HASH_LENGTH, }; @@ -497,7 +497,7 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ estimated_costs::worst_case_costs::add_worst_case_get_merk_node, - merk::{NodeType, TreeType}, + merk::{tree_type::TreeType, NodeType}, test_utils::{empty_path_merk, empty_path_merk_read_only, make_batch_seq}, tree::kv::ValueDefinedCostType, }; diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 0bc1eb31a..48f067a83 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -186,7 +186,7 @@ use grovedb_merk::{ tree::{combine_hash, value_hash}, BatchEntry, CryptoHash, KVIterator, Merk, }; -use grovedb_merk::{merk::TreeType, tree::AggregateData}; +use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData}; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 1b018dbfa..70d1683bb 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -8,7 +8,7 @@ use grovedb_merk::{ average_case_costs::EstimatedLayerInformation, worst_case_costs::add_average_case_cost_for_is_empty_tree_except, }, - merk::TreeType, + merk::tree_type::TreeType, HASH_LENGTH_U32, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; diff --git a/grovedb/src/operations/delete/delete_up_tree.rs b/grovedb/src/operations/delete/delete_up_tree.rs index e2ca1dbe2..c4dea4304 100644 --- a/grovedb/src/operations/delete/delete_up_tree.rs +++ b/grovedb/src/operations/delete/delete_up_tree.rs @@ -5,7 +5,7 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::tree_type::TreeType; use grovedb_path::SubtreePath; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 01d2252f0..ae5937371 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -18,7 +18,7 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{merk::TreeType, proofs::Query, KVIterator}; +use grovedb_merk::{merk::tree_type::TreeType, proofs::Query, KVIterator}; #[cfg(feature = "full")] use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; use grovedb_path::SubtreePath; diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 09f9da76c..d57eebb7b 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -5,7 +5,7 @@ use grovedb_costs::{ }; use grovedb_merk::{ estimated_costs::worst_case_costs::add_worst_case_cost_for_is_empty_tree_except, - merk::TreeType, tree::kv::KV, + merk::tree_type::TreeType, tree::kv::KV, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; use grovedb_version::{ diff --git a/grovedb/src/operations/get/average_case.rs b/grovedb/src/operations/get/average_case.rs index e05e63b1d..6cac07df9 100644 --- a/grovedb/src/operations/get/average_case.rs +++ b/grovedb/src/operations/get/average_case.rs @@ -2,7 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::OperationCost; -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::tree_type::TreeType; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; diff --git a/grovedb/src/operations/get/worst_case.rs b/grovedb/src/operations/get/worst_case.rs index 663e74bbf..4705558ef 100644 --- a/grovedb/src/operations/get/worst_case.rs +++ b/grovedb/src/operations/get/worst_case.rs @@ -2,7 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::OperationCost; -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::tree_type::TreeType; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 37a56d643..99c0e40ad 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -2,7 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; -use grovedb_merk::merk::TreeType; +use grovedb_merk::merk::tree_type::TreeType; use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_version::error::GroveVersionError; diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 5f167719c..b9eeac085 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -2,7 +2,7 @@ mod state_sync_session; use std::pin::Pin; -use grovedb_merk::{merk::TreeType, tree::hash::CryptoHash, ChunkProducer}; +use grovedb_merk::{merk::tree_type::TreeType, tree::hash::CryptoHash, ChunkProducer}; use grovedb_path::SubtreePath; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -256,7 +256,7 @@ impl GroveDb { pub(crate) mod utils { use grovedb_merk::{ ed::Encode, - merk::TreeType, + merk::tree_type::TreeType, proofs::{Decoder, Op}, }; diff --git a/grovedb/src/replication/state_sync_session.rs b/grovedb/src/replication/state_sync_session.rs index 79defcfde..de44d8455 100644 --- a/grovedb/src/replication/state_sync_session.rs +++ b/grovedb/src/replication/state_sync_session.rs @@ -6,7 +6,7 @@ use std::{ }; use grovedb_merk::{ - merk::TreeType, + merk::tree_type::TreeType, tree::{kv::ValueDefinedCostType, value_hash}, CryptoHash, Restorer, }; diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 9ff58aef5..e23188700 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -6,7 +6,7 @@ use grovedb_version::{error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; -use crate::merk::{NodeType, TreeType}; +use crate::merk::{tree_type::TreeType, NodeType}; #[cfg(feature = "full")] use crate::{ error::Error, diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index 5ad58c0a5..ee6d157e6 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -5,7 +5,7 @@ use grovedb_costs::OperationCost; #[cfg(feature = "full")] use integer_encoding::VarInt; -use crate::merk::{NodeType, TreeType}; +use crate::merk::{tree_type::TreeType, NodeType}; #[cfg(feature = "full")] use crate::{tree::kv::KV, HASH_BLOCK_SIZE_U32, HASH_LENGTH_U32}; diff --git a/merk/src/lib.rs b/merk/src/lib.rs index cd74d7acd..d6849d7b7 100644 --- a/merk/src/lib.rs +++ b/merk/src/lib.rs @@ -84,7 +84,8 @@ pub use tree::{CryptoHash, TreeFeatureType}; pub use crate::merk::{ defaults::ROOT_KEY_KEY, prove::{ProofConstructionResult, ProofWithoutEncodingResult}, - IsSumTree, KVIterator, Merk, MerkType, RootHashKeyAndAggregateData, + tree_type::TreeType, + KVIterator, Merk, MerkType, RootHashKeyAndAggregateData, }; #[cfg(feature = "full")] pub use crate::visualize::VisualizeableMerk; diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index bcf537802..b2ae3cdec 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -41,6 +41,7 @@ pub mod open; pub mod prove; pub mod restore; pub mod source; +pub mod tree_type; use std::{ cell::Cell, @@ -57,6 +58,7 @@ use grovedb_costs::{ use grovedb_storage::{self, Batch, RawIterator, StorageContext}; use grovedb_version::version::GroveVersion; use source::MerkSource; +use tree_type::TreeType; use crate::{ error::Error, @@ -76,7 +78,6 @@ use crate::{ Error::{CostsError, EdError, StorageError}, Link, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, - TreeFeatureType, }; /// Key update types @@ -112,9 +113,6 @@ pub type BatchValue = ( KeyValueStorageCost, ); -/// A bool type -pub type IsSumTree = bool; - /// Root hash key and sum pub type RootHashKeyAndAggregateData = (CryptoHash, Option>, AggregateData); @@ -244,74 +242,6 @@ impl MerkType { } } } -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum TreeType { - NormalTree = 0, - SumTree = 1, - BigSumTree = 2, - CountTree = 3, - CountSumTree = 4, -} - -impl TryFrom for TreeType { - type Error = Error; - - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(TreeType::NormalTree), - 1 => Ok(TreeType::SumTree), - 2 => Ok(TreeType::BigSumTree), - 3 => Ok(TreeType::CountTree), - 3 => Ok(TreeType::CountSumTree), - n => Err(Error::UnknownTreeType(format!("got {}, max is 3", n))), // Error handling - } - } -} - -impl fmt::Display for TreeType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match *self { - TreeType::NormalTree => "Normal Tree", - TreeType::SumTree => "Sum Tree", - TreeType::BigSumTree => "Big Sum Tree", - TreeType::CountTree => "Count Tree", - TreeType::CountSumTree => "Count Sum Tree", - }; - write!(f, "{}", s) - } -} - -impl TreeType { - pub fn allows_sum_item(&self) -> bool { - match self { - TreeType::NormalTree => false, - TreeType::SumTree => true, - TreeType::BigSumTree => true, - TreeType::CountTree => false, - TreeType::CountSumTree => true, - } - } - - pub fn inner_node_type(&self) -> NodeType { - match self { - TreeType::NormalTree => NodeType::NormalNode, - TreeType::SumTree => NodeType::SumNode, - TreeType::BigSumTree => NodeType::BigSumNode, - TreeType::CountTree => NodeType::CountNode, - TreeType::CountSumTree => NodeType::CountSumNode, - } - } - - pub fn empty_tree_feature_type(&self) -> TreeFeatureType { - match self { - TreeType::NormalTree => TreeFeatureType::BasicMerkNode, - TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), - TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), - TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), - TreeType::CountSumTree => TreeFeatureType::CountedSummedMerkNode(0, 0), - } - } -} #[cfg(any(feature = "full", feature = "verify"))] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -872,12 +802,14 @@ mod test { use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use super::{Merk, RefWalker, TreeType}; + use super::{Merk, RefWalker}; use crate::{ - merk::source::MerkSource, test_utils::*, tree::kv::ValueDefinedCostType, Op, + merk::{source::MerkSource, tree_type::TreeType}, + test_utils::*, + tree::kv::ValueDefinedCostType, + Op, TreeFeatureType::BasicMerkNode, }; - // TODO: Close and then reopen test fn assert_invariants(merk: &TempMerk) { diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index fa172bf24..302e9e890 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -5,7 +5,7 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ - merk::TreeType, + merk::tree_type::TreeType, tree::kv::ValueDefinedCostType, Error, Merk, MerkType, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, @@ -103,7 +103,8 @@ mod test { use tempfile::TempDir; use crate::{ - merk::TreeType, tree::kv::ValueDefinedCostType, Merk, Op, TreeFeatureType::BasicMerkNode, + merk::tree_type::TreeType, tree::kv::ValueDefinedCostType, Merk, Op, + TreeFeatureType::BasicMerkNode, }; #[test] diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index 5fb91b8be..b5a75e375 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -36,7 +36,7 @@ use grovedb_version::version::GroveVersion; use crate::{ merk, - merk::{MerkSource, TreeType}, + merk::{tree_type::TreeType, MerkSource}, proofs::{ chunk::{ chunk::{LEFT, RIGHT}, @@ -561,7 +561,7 @@ mod tests { use super::*; use crate::{ - merk::{chunks::ChunkProducer, TreeType}, + merk::{chunks::ChunkProducer, tree_type::TreeType}, proofs::chunk::{ chunk::tests::traverse_get_node_hash, error::ChunkError::InvalidChunkProof, }, diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs index 14c707958..a4f96027e 100644 --- a/merk/src/merk/source.rs +++ b/merk/src/merk/source.rs @@ -3,7 +3,7 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ - merk::TreeType, + merk::tree_type::TreeType, tree::{kv::ValueDefinedCostType, Fetch, TreeNode}, Error, Link, Merk, }; diff --git a/merk/src/merk/tree_type.rs b/merk/src/merk/tree_type.rs new file mode 100644 index 000000000..cf1a6a3dd --- /dev/null +++ b/merk/src/merk/tree_type.rs @@ -0,0 +1,72 @@ +use std::fmt; + +use crate::{merk::NodeType, Error, TreeFeatureType}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum TreeType { + NormalTree = 0, + SumTree = 1, + BigSumTree = 2, + CountTree = 3, + CountSumTree = 4, +} + +impl TryFrom for TreeType { + type Error = Error; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(TreeType::NormalTree), + 1 => Ok(TreeType::SumTree), + 2 => Ok(TreeType::BigSumTree), + 3 => Ok(TreeType::CountTree), + 4 => Ok(TreeType::CountSumTree), + n => Err(Error::UnknownTreeType(format!("got {}, max is 4", n))), // Error handling + } + } +} + +impl fmt::Display for TreeType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match *self { + TreeType::NormalTree => "Normal Tree", + TreeType::SumTree => "Sum Tree", + TreeType::BigSumTree => "Big Sum Tree", + TreeType::CountTree => "Count Tree", + TreeType::CountSumTree => "Count Sum Tree", + }; + write!(f, "{}", s) + } +} + +impl TreeType { + pub fn allows_sum_item(&self) -> bool { + match self { + TreeType::NormalTree => false, + TreeType::SumTree => true, + TreeType::BigSumTree => true, + TreeType::CountTree => false, + TreeType::CountSumTree => true, + } + } + + pub fn inner_node_type(&self) -> NodeType { + match self { + TreeType::NormalTree => NodeType::NormalNode, + TreeType::SumTree => NodeType::SumNode, + TreeType::BigSumTree => NodeType::BigSumNode, + TreeType::CountTree => NodeType::CountNode, + TreeType::CountSumTree => NodeType::CountSumNode, + } + } + + pub fn empty_tree_feature_type(&self) -> TreeFeatureType { + match self { + TreeType::NormalTree => TreeFeatureType::BasicMerkNode, + TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), + TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), + TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), + TreeType::CountSumTree => TreeFeatureType::CountedSummedMerkNode(0, 0), + } + } +} diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index f53ce730c..bdfa3219c 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -40,7 +40,7 @@ use rand::prelude::*; pub use temp_merk::TempMerk; use crate::{ - merk::TreeType, + merk::tree_type::TreeType, tree::{ kv::{ValueDefinedCostType, KV}, BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, TreeNode, Walker, diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 93cff86f5..63bf6a39f 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -42,7 +42,7 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use crate::Merk; -use crate::{merk::TreeType, tree::kv::ValueDefinedCostType}; +use crate::{merk::tree_type::TreeType, tree::kv::ValueDefinedCostType}; #[cfg(feature = "full")] /// Wraps a Merk instance and deletes it from disk it once it goes out of scope. diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 8c9577bcc..c43287d7c 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -15,7 +15,7 @@ use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(any(feature = "full", feature = "verify"))] use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; use crate::{ - merk::{NodeType, TreeType}, + merk::{tree_type::TreeType, NodeType}, TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, }; From 64f26bf5f548c411839aacc33a41235d56454cb6 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 06:11:56 +0700 Subject: [PATCH 12/30] cleanup --- grovedb/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 48f067a83..a9c0becad 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -186,7 +186,8 @@ use grovedb_merk::{ tree::{combine_hash, value_hash}, BatchEntry, CryptoHash, KVIterator, Merk, }; -use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData}; +#[cfg(any(feature = "full", feature = "verify"))] +pub use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData}; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] From 35c64c3c6091f5d7cc29c7b5deafae7a144a856f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 06:37:34 +0700 Subject: [PATCH 13/30] fix --- grovedb/src/element/helpers.rs | 18 ++++++++++++++---- grovedb/src/lib.rs | 2 +- .../src/operations/delete/delete_up_tree.rs | 6 +++--- grovedb/src/operations/delete/mod.rs | 10 +++++----- merk/src/lib.rs | 2 +- merk/src/merk/tree_type.rs | 6 ++++++ 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 09775d483..39224f3af 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -6,10 +6,7 @@ use grovedb_merk::tree::kv::{ ValueDefinedCostType, ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, }; -use grovedb_merk::{ - merk::{tree_type::TreeType, NodeType}, - TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, -}; +use grovedb_merk::{MaybeTree, merk::{tree_type::TreeType, NodeType}, TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}}; #[cfg(feature = "full")] use grovedb_merk::{ tree::{kv::KV, TreeNode}, @@ -203,6 +200,19 @@ impl Element { } } + #[cfg(any(feature = "full", feature = "verify"))] + /// Check if the element is a tree and return the tree type + pub fn maybe_tree_type(&self) -> MaybeTree { + match self { + Element::Tree(..) => MaybeTree::Tree(TreeType::NormalTree), + Element::SumTree(..) => MaybeTree::Tree(TreeType::SumTree), + Element::BigSumTree(..) => MaybeTree::Tree(TreeType::BigSumTree), + Element::CountTree(..) => MaybeTree::Tree(TreeType::CountTree), + Element::CountSumTree(..) => MaybeTree::Tree(TreeType::CountSumTree), + _ => MaybeTree::NotTree, + } + } + #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a big sum tree pub fn is_big_sum_tree(&self) -> bool { diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a9c0becad..9403870e6 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -187,7 +187,7 @@ use grovedb_merk::{ BatchEntry, CryptoHash, KVIterator, Merk, }; #[cfg(any(feature = "full", feature = "verify"))] -pub use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData}; +pub use grovedb_merk::{merk::tree_type::{TreeType, MaybeTree}, tree::AggregateData}; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] diff --git a/grovedb/src/operations/delete/delete_up_tree.rs b/grovedb/src/operations/delete/delete_up_tree.rs index c4dea4304..6c747c16b 100644 --- a/grovedb/src/operations/delete/delete_up_tree.rs +++ b/grovedb/src/operations/delete/delete_up_tree.rs @@ -5,7 +5,7 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::merk::tree_type::TreeType; +use grovedb_merk::MaybeTree; use grovedb_path::SubtreePath; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, @@ -170,7 +170,7 @@ impl GroveDb { path: SubtreePath, key: &[u8], options: &DeleteUpTreeOptions, - is_known_to_be_subtree: Option, + is_known_to_be_subtree: Option, mut current_batch_operations: Vec, transaction: TransactionArg, grove_version: &GroveVersion, @@ -202,7 +202,7 @@ impl GroveDb { path: SubtreePath, key: &[u8], options: &DeleteUpTreeOptions, - is_known_to_be_subtree: Option, + is_known_to_be_subtree: Option, current_batch_operations: &mut Vec, transaction: TransactionArg, grove_version: &GroveVersion, diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index ae5937371..a2082c9ac 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -18,7 +18,7 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{merk::tree_type::TreeType, proofs::Query, KVIterator}; +use grovedb_merk::{merk::tree_type::TreeType, proofs::Query, KVIterator, MaybeTree}; #[cfg(feature = "full")] use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; use grovedb_path::SubtreePath; @@ -511,7 +511,7 @@ impl GroveDb { path: SubtreePath, key: &[u8], options: &DeleteOptions, - is_known_to_be_subtree: Option, + is_known_to_be_subtree: Option, current_batch_operations: &[QualifiedGroveDbOp], transaction: TransactionArg, grove_version: &GroveVersion, @@ -550,12 +550,12 @@ impl GroveDb { &mut cost, self.get_raw(path.clone(), key.as_ref(), transaction, grove_version) ); - element.tree_type() + element.maybe_tree_type() } - Some(x) => Some(x), + Some(x) => x, }; - if let Some(tree_type) = tree_type { + if let MaybeTree::Tree(tree_type) = tree_type { let subtree_merk_path = path.derive_owned_with_child(key); let subtree_merk_path_vec = subtree_merk_path.to_vec(); let batch_deleted_keys = current_batch_operations diff --git a/merk/src/lib.rs b/merk/src/lib.rs index d6849d7b7..69580b308 100644 --- a/merk/src/lib.rs +++ b/merk/src/lib.rs @@ -84,7 +84,7 @@ pub use tree::{CryptoHash, TreeFeatureType}; pub use crate::merk::{ defaults::ROOT_KEY_KEY, prove::{ProofConstructionResult, ProofWithoutEncodingResult}, - tree_type::TreeType, + tree_type::TreeType, tree_type::MaybeTree, KVIterator, Merk, MerkType, RootHashKeyAndAggregateData, }; #[cfg(feature = "full")] diff --git a/merk/src/merk/tree_type.rs b/merk/src/merk/tree_type.rs index cf1a6a3dd..0a381c822 100644 --- a/merk/src/merk/tree_type.rs +++ b/merk/src/merk/tree_type.rs @@ -2,6 +2,12 @@ use std::fmt; use crate::{merk::NodeType, Error, TreeFeatureType}; +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum MaybeTree { + Tree(TreeType), + NotTree, +} + #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum TreeType { NormalTree = 0, From 2417f7a72900cd3dca943ad52c979bc8abfdaa20 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 06:37:40 +0700 Subject: [PATCH 14/30] fix --- grovedb/src/element/helpers.rs | 6 +++++- grovedb/src/lib.rs | 5 ++++- merk/src/lib.rs | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 39224f3af..1c156e41a 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -6,7 +6,11 @@ use grovedb_merk::tree::kv::{ ValueDefinedCostType, ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, }; -use grovedb_merk::{MaybeTree, merk::{tree_type::TreeType, NodeType}, TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}}; +use grovedb_merk::{ + merk::{tree_type::TreeType, NodeType}, + MaybeTree, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, +}; #[cfg(feature = "full")] use grovedb_merk::{ tree::{kv::KV, TreeNode}, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 9403870e6..e3a92d9e1 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -187,7 +187,10 @@ use grovedb_merk::{ BatchEntry, CryptoHash, KVIterator, Merk, }; #[cfg(any(feature = "full", feature = "verify"))] -pub use grovedb_merk::{merk::tree_type::{TreeType, MaybeTree}, tree::AggregateData}; +pub use grovedb_merk::{ + merk::tree_type::{MaybeTree, TreeType}, + tree::AggregateData, +}; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] diff --git a/merk/src/lib.rs b/merk/src/lib.rs index 69580b308..c5814634e 100644 --- a/merk/src/lib.rs +++ b/merk/src/lib.rs @@ -84,7 +84,8 @@ pub use tree::{CryptoHash, TreeFeatureType}; pub use crate::merk::{ defaults::ROOT_KEY_KEY, prove::{ProofConstructionResult, ProofWithoutEncodingResult}, - tree_type::TreeType, tree_type::MaybeTree, + tree_type::MaybeTree, + tree_type::TreeType, KVIterator, Merk, MerkType, RootHashKeyAndAggregateData, }; #[cfg(feature = "full")] From 558bd555ed9aa9e7cf8509cdd73a910a80a286b3 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 10 Jan 2025 08:41:30 +0700 Subject: [PATCH 15/30] added helpers --- grovedb/src/element/constructor.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 3dc346b3b..83e2ac432 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -41,12 +41,36 @@ impl Element { Element::new_count_tree(Default::default()) } + #[cfg(feature = "full")] + /// Set element to default empty count sum tree without flags + pub fn empty_count_sum_tree() -> Self { + Element::new_count_sum_tree(Default::default()) + } + #[cfg(feature = "full")] /// Set element to default empty sum tree with flags pub fn empty_sum_tree_with_flags(flags: Option) -> Self { Element::new_sum_tree_with_flags(Default::default(), flags) } + #[cfg(feature = "full")] + /// Set element to default empty sum tree with flags + pub fn empty_big_sum_tree_with_flags(flags: Option) -> Self { + Element::new_big_sum_tree_with_flags(Default::default(), flags) + } + + #[cfg(feature = "full")] + /// Set element to default empty count tree with flags + pub fn empty_count_tree_with_flags(flags: Option) -> Self { + Element::new_count_tree_with_flags(Default::default(), flags) + } + + #[cfg(feature = "full")] + /// Set element to default empty count sum tree with flags + pub fn empty_count_sum_tree_with_flags(flags: Option) -> Self { + Element::new_count_sum_tree_with_flags(Default::default(), flags) + } + #[cfg(feature = "full")] /// Set element to an item without flags pub fn new_item(item_value: Vec) -> Self { From 263dcd5164c7db597bf7de7b6de133ad7cd77d50 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 11 Jan 2025 02:30:31 +0700 Subject: [PATCH 16/30] more fixes --- grovedb-version/src/lib.rs | 17 +++ grovedb-version/src/version/merk_versions.rs | 1 + grovedb-version/src/version/v1.rs | 1 + grovedb-version/src/version/v2.rs | 1 + .../estimated_costs/average_case_costs.rs | 3 + .../src/estimated_costs/average_case_costs.rs | 2 +- grovedb/src/operations/delete/average_case.rs | 4 +- grovedb/src/operations/delete/mod.rs | 2 +- .../src/estimated_costs/average_case_costs.rs | 126 ++++++++++++++---- merk/src/merk/tree_type.rs | 2 +- merk/src/tree/mod.rs | 2 +- 11 files changed, 132 insertions(+), 29 deletions(-) diff --git a/grovedb-version/src/lib.rs b/grovedb-version/src/lib.rs index 48b80a52e..f66019d4a 100644 --- a/grovedb-version/src/lib.rs +++ b/grovedb-version/src/lib.rs @@ -34,6 +34,23 @@ macro_rules! check_grovedb_v0 { }}; } +#[macro_export] +macro_rules! check_grovedb_v0_or_v1 { + ($method:expr, $version:expr) => {{ + const EXPECTED_VERSION_V0: u16 = 0; + const EXPECTED_VERSION_V1: u16 = 1; + if $version != EXPECTED_VERSION_V0 && $version != EXPECTED_VERSION_V1 { + return Err(GroveVersionError::UnknownVersionMismatch { + method: $method.to_string(), + known_versions: vec![EXPECTED_VERSION_V0, EXPECTED_VERSION_V1], + received: $version, + } + .into()); + } + $version + }}; +} + #[macro_export] macro_rules! check_merk_v0_with_cost { ($method:expr, $version:expr) => {{ diff --git a/grovedb-version/src/version/merk_versions.rs b/grovedb-version/src/version/merk_versions.rs index f92d90132..d0d122da2 100644 --- a/grovedb-version/src/version/merk_versions.rs +++ b/grovedb-version/src/version/merk_versions.rs @@ -8,4 +8,5 @@ pub struct MerkVersions { #[derive(Clone, Debug, Default)] pub struct MerkAverageCaseCostsVersions { pub add_average_case_merk_propagate: FeatureVersion, + pub sum_tree_estimated_size: FeatureVersion, } diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 71a1c71f3..5bf58180a 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -187,6 +187,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion { merk_versions: MerkVersions { average_case_costs: MerkAverageCaseCostsVersions { add_average_case_merk_propagate: 0, + sum_tree_estimated_size: 0, }, }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 6bb0e31ee..3591ba1af 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -187,6 +187,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { merk_versions: MerkVersions { average_case_costs: MerkAverageCaseCostsVersions { add_average_case_merk_propagate: 1, // changed + sum_tree_estimated_size: 1, // changed }, }, }; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 52b42fc65..40392bb9c 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -712,6 +712,9 @@ mod tests { 1, SomeSumTrees { sum_trees_weight: 1, + big_sum_trees_weight: 0, + count_trees_weight: 0, + count_sum_trees_weight: 0, non_sum_trees_weight: 1, }, None, diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index bb0824cb4..21b1ea035 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -442,7 +442,7 @@ impl GroveDb { &cost, estimated_layer_information .estimated_layer_sizes - .value_with_feature_and_flags_size() + .value_with_feature_and_flags_size(grove_version) .map_err(Error::MerkError) ); add_average_case_merk_delete(&mut cost, key_len, value_size); diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 70d1683bb..9ef48c1c7 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -75,7 +75,7 @@ impl GroveDb { &cost, layer_info .estimated_layer_sizes - .value_with_feature_and_flags_size() + .value_with_feature_and_flags_size(grove_version) .map_err(Error::MerkError) ); Ok(( @@ -100,7 +100,7 @@ impl GroveDb { &cost, layer_info .estimated_layer_sizes - .subtree_with_feature_and_flags_size() + .subtree_with_feature_and_flags_size(grove_version) .map_err(Error::MerkError) ); Ok(( diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index a2082c9ac..d07d628f4 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -18,7 +18,7 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{merk::tree_type::TreeType, proofs::Query, KVIterator, MaybeTree}; +use grovedb_merk::{proofs::Query, KVIterator, MaybeTree}; #[cfg(feature = "full")] use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; use grovedb_path::SubtreePath; diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index e23188700..7a5e93e91 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -2,7 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::{CostResult, CostsExt, OperationCost}; -use grovedb_version::{error::GroveVersionError, version::GroveVersion}; +use grovedb_version::{check_grovedb_v0_or_v1, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; @@ -40,26 +40,88 @@ pub enum EstimatedSumTrees { SomeSumTrees { /// Sum trees weight sum_trees_weight: Weight, + /// Big Sum trees weight + big_sum_trees_weight: Weight, + /// Count trees weight + count_trees_weight: Weight, + /// Count Sum trees weight + count_sum_trees_weight: Weight, /// Non sum trees weight non_sum_trees_weight: Weight, }, /// All sum trees AllSumTrees, + /// All big sum trees + AllBigSumTrees, + /// All count trees + AllCountTrees, + /// All count sum trees + AllCountSumTrees, } -#[cfg(feature = "full")] #[cfg(feature = "full")] impl EstimatedSumTrees { - fn estimated_size(&self) -> Result { + fn estimated_size(&self, grove_version: &GroveVersion) -> Result { + let version = check_grovedb_v0_or_v1!( + "EstimatedSumTrees::estimated_size", + grove_version + .merk_versions + .average_case_costs + .sum_tree_estimated_size + ); match self { EstimatedSumTrees::NoSumTrees => Ok(0), EstimatedSumTrees::SomeSumTrees { sum_trees_weight, + big_sum_trees_weight, + count_trees_weight, + count_sum_trees_weight, non_sum_trees_weight, - } => (*non_sum_trees_weight as u32 * 9) - .checked_div(*sum_trees_weight as u32 + *non_sum_trees_weight as u32) - .ok_or(Error::DivideByZero("weights add up to 0")), - EstimatedSumTrees::AllSumTrees => Ok(8), + } => { + // Example calculation including new weights + let total_weight = *sum_trees_weight as u32 + + *big_sum_trees_weight as u32 + + *count_trees_weight as u32 + + *count_sum_trees_weight as u32 + + *non_sum_trees_weight as u32; + if total_weight == 0 { + return Err(Error::DivideByZero("weights add up to 0")); + }; + if version == 0 { + Ok((*non_sum_trees_weight as u32 * 9) + / (*sum_trees_weight as u32 + *non_sum_trees_weight as u32)) + } else if version == 1 { + let estimated_size = (*sum_trees_weight as u32 + * TreeType::SumTree.inner_node_type().cost()) + .checked_add( + *big_sum_trees_weight as u32 + * TreeType::BigSumTree.inner_node_type().cost(), + ) + .and_then(|sum| { + sum.checked_add( + *count_trees_weight as u32 + * TreeType::CountTree.inner_node_type().cost(), + ) + }) + .and_then(|sum| { + sum.checked_add( + *count_sum_trees_weight as u32 + * TreeType::CountSumTree.inner_node_type().cost(), + ) + }) + .ok_or(Error::Overflow("Estimated size calculation overflowed"))?; + + Ok(estimated_size / total_weight) + } else { + Err(Error::CorruptedCodeExecution("we already checked versions")) + } + } + EstimatedSumTrees::AllSumTrees => Ok(TreeType::SumTree.inner_node_type().cost()), + EstimatedSumTrees::AllBigSumTrees => Ok(TreeType::BigSumTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountTrees => Ok(TreeType::CountTree.inner_node_type().cost()), + EstimatedSumTrees::AllCountSumTrees => { + Ok(TreeType::CountSumTree.inner_node_type().cost()) + } } } } @@ -128,19 +190,26 @@ impl EstimatedLayerSizes { /// Returns the size of a subtree's feature and flags /// This only takes into account subtrees in the estimated layer info /// Only should be used when it is known to be a subtree - pub fn subtree_with_feature_and_flags_size(&self) -> Result { + pub fn subtree_with_feature_and_flags_size( + &self, + grove_version: &GroveVersion, + ) -> Result { match self { EstimatedLayerSizes::AllSubtrees(_, estimated_sum_trees, flags_size) => { // 1 for enum type // 1 for empty // 1 for flags size - Ok(estimated_sum_trees.estimated_size()? + flags_size.unwrap_or_default() + 3) + Ok(estimated_sum_trees.estimated_size(grove_version)? + + flags_size.unwrap_or_default() + + 3) } EstimatedLayerSizes::Mix { subtrees_size, .. } => match subtrees_size { None => Err(Error::WrongEstimatedCostsElementTypeForLevel( "this layer is a mix but doesn't have subtrees", )), - Some((_, est, fs, _)) => Ok(est.estimated_size()? + fs.unwrap_or_default() + 3), + Some((_, est, fs, _)) => { + Ok(est.estimated_size(grove_version)? + fs.unwrap_or_default() + 3) + } }, _ => Err(Error::WrongEstimatedCostsElementTypeForLevel( "this layer needs to have trees", @@ -149,7 +218,10 @@ impl EstimatedLayerSizes { } /// Returns the size of a value's feature and flags - pub fn value_with_feature_and_flags_size(&self) -> Result { + pub fn value_with_feature_and_flags_size( + &self, + grove_version: &GroveVersion, + ) -> Result { match self { EstimatedLayerSizes::AllItems(_, average_value_size, flags_size) => { // 1 for enum type @@ -168,7 +240,9 @@ impl EstimatedLayerSizes { // 1 for enum type // 1 for empty // 1 for flags size - Ok(estimated_sum_trees.estimated_size()? + flags_size.unwrap_or_default() + 3) + Ok(estimated_sum_trees.estimated_size(grove_version)? + + flags_size.unwrap_or_default() + + 3) } EstimatedLayerSizes::Mix { subtrees_size, @@ -188,7 +262,7 @@ impl EstimatedLayerSizes { let (subtree_size, subtree_weight) = match subtrees_size { None => None, Some((_, est, fs, weight)) => Some(( - est.estimated_size()? + fs.unwrap_or_default() + 3, + est.estimated_size(grove_version)? + fs.unwrap_or_default() + 3, *weight as u32, )), } @@ -416,8 +490,8 @@ pub fn add_average_case_merk_propagate( .average_case_costs .add_average_case_merk_propagate { - 0 => add_average_case_merk_propagate_v0(cost, input), - 1 => add_average_case_merk_propagate_v1(cost, input), + 0 => add_average_case_merk_propagate_v0(cost, input, grove_version), + 1 => add_average_case_merk_propagate_v1(cost, input, grove_version), version => Err(Error::VersionError( GroveVersionError::UnknownVersionMismatch { method: "add_average_case_merk_propagate".to_string(), @@ -432,6 +506,7 @@ pub fn add_average_case_merk_propagate( fn add_average_case_merk_propagate_v1( cost: &mut OperationCost, input: &EstimatedLayerInformation, + grove_version: &GroveVersion, ) -> Result<(), Error> { let mut nodes_updated = 0; // Propagation requires to recompute and write hashes up to the root @@ -464,7 +539,7 @@ fn add_average_case_merk_propagate_v1( .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); // in order to simplify calculations we get the estimated size and remove the // cost for the basic merk - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; nodes_updated * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, @@ -515,7 +590,8 @@ fn add_average_case_merk_propagate_v1( Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { let flags_len = average_flags_size.unwrap_or(0); let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len, @@ -579,7 +655,7 @@ fn add_average_case_merk_propagate_v1( ) => { let flags_len = average_flags_size.unwrap_or(0); let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; nodes_updated * KV::layered_node_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, @@ -631,7 +707,8 @@ fn add_average_case_merk_propagate_v1( |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { let flags_len = average_flags_size.unwrap_or(0); let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len + sum_tree_addition, @@ -700,6 +777,7 @@ fn add_average_case_merk_propagate_v1( fn add_average_case_merk_propagate_v0( cost: &mut OperationCost, input: &EstimatedLayerInformation, + grove_version: &GroveVersion, ) -> Result<(), Error> { let mut nodes_updated = 0; // Propagation requires to recompute and write hashes up to the root @@ -732,7 +810,7 @@ fn add_average_case_merk_propagate_v0( .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); // in order to simplify calculations we get the estimated size and remove the // cost for the basic merk - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; nodes_updated * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, @@ -783,7 +861,8 @@ fn add_average_case_merk_propagate_v0( Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { let flags_len = average_flags_size.unwrap_or(0); let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len, @@ -847,7 +926,7 @@ fn add_average_case_merk_propagate_v0( ) => { let flags_len = average_flags_size.unwrap_or(0); let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = estimated_sum_trees.estimated_size(grove_version)?; nodes_updated * KV::layered_node_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, @@ -899,7 +978,8 @@ fn add_average_case_merk_propagate_v0( |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { let flags_len = average_flags_size.unwrap_or(0); let value_len = LAYER_COST_SIZE + flags_len; - let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let sum_tree_addition = + estimated_sum_trees.estimated_size(grove_version)?; let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len + sum_tree_addition, diff --git a/merk/src/merk/tree_type.rs b/merk/src/merk/tree_type.rs index 0a381c822..ef845f21a 100644 --- a/merk/src/merk/tree_type.rs +++ b/merk/src/merk/tree_type.rs @@ -56,7 +56,7 @@ impl TreeType { } } - pub fn inner_node_type(&self) -> NodeType { + pub const fn inner_node_type(&self) -> NodeType { match self { TreeType::NormalTree => NodeType::NormalNode, TreeType::SumTree => NodeType::SumNode, diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 8a10f82b0..5ce96543e 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -526,7 +526,7 @@ impl TreeNode { match self.link(left) { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => Ok(0), - AggregateData::Sum(s) => Ok(0), + AggregateData::Sum(_) => Ok(0), AggregateData::BigSum(_) => Ok(0), AggregateData::Count(c) => Ok(c), AggregateData::CountAndSum(c, _) => Ok(c), From a25cc077d2e425cab7dcba41e4611444857f62d7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 13 Jan 2025 06:19:59 +0700 Subject: [PATCH 17/30] better error when key not found --- grovedb/src/element/get.rs | 65 ++++++++++++++++++++++++------- grovedb/src/element/insert.rs | 48 +++++++++++++++++------ grovedb/src/element/query.rs | 11 +++++- grovedb/src/operations/get/mod.rs | 41 ++++++++++++++++--- grovedb/src/tests/mod.rs | 36 ++++++++++++----- path/Cargo.toml | 1 + path/src/subtree_path.rs | 50 +++++++++++++++++++++++- path/src/util/compact_bytes.rs | 2 +- 8 files changed, 210 insertions(+), 44 deletions(-) diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 66a9b3ca7..02ed9f95b 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -17,7 +17,10 @@ use grovedb_version::{ }; use integer_encoding::VarInt; -use crate::element::{CostSize, SUM_ITEM_COST_SIZE}; +use crate::{ + element::{CostSize, SUM_ITEM_COST_SIZE}, + operations::proof::util::path_as_slices_hex_to_ascii, +}; #[cfg(feature = "full")] use crate::{Element, Error, Hash}; @@ -29,19 +32,31 @@ impl Element { merk: &Merk, key: K, allow_cache: bool, + extra_error_info: Option String>, grove_version: &GroveVersion, ) -> CostResult { check_grovedb_v0_with_cost!("get", grove_version.grovedb_versions.element.get); Self::get_optional(merk, key.as_ref(), allow_cache, grove_version).map(|result| { let value = result?; value.ok_or_else(|| { + let key_single_byte = if key.as_ref().len() == 1 { + format!("({} in decimal) ", key.as_ref().get(0).unwrap()) + } else { + String::new() + }; + let extra_error_info_string = extra_error_info + .map(|callback| format!(" {}", callback())) + .unwrap_or(String::new()); Error::PathKeyNotFound(format!( - "get: key \"{}\" not found in Merk that has a root key [{}] and is of type {}", + "get: key 0x{} {}not found in Merk that has a root key [{}] and is of type \ + {}{}", hex::encode(key), + key_single_byte, merk.root_key() .map(hex::encode) .unwrap_or("None".to_string()), - merk.merk_type + merk.merk_type, + extra_error_info_string, )) }) }) @@ -322,7 +337,13 @@ impl Element { let element = cost_return_on_error!( &mut cost, - Self::get(merk, key.as_ref(), allow_cache, grove_version) + Self::get( + merk, + key.as_ref(), + allow_cache, + Some(|| { format!("path is {}", path_as_slices_hex_to_ascii(path)) }), + grove_version + ) ); let absolute_element = cost_return_on_error_no_add!( @@ -414,9 +435,15 @@ mod tests { .unwrap(); assert_eq!( - Element::get(&merk, b"another-key", true, grove_version) - .unwrap() - .expect("expected successful get"), + Element::get( + &merk, + b"another-key", + true, + None:: String>, + grove_version + ) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value".to_vec()), ); @@ -426,12 +453,24 @@ mod tests { .unwrap() .expect("expected successful insertion 2"); - let cost_with_cache = Element::get(&merk, b"another-key", true, grove_version) - .cost_as_result() - .expect("expected to get cost"); - let cost_without_cache = Element::get(&merk, b"another-key", false, grove_version) - .cost_as_result() - .expect("expected to get cost"); + let cost_with_cache = Element::get( + &merk, + b"another-key", + true, + None:: String>, + grove_version, + ) + .cost_as_result() + .expect("expected to get cost"); + let cost_without_cache = Element::get( + &merk, + b"another-key", + false, + None:: String>, + grove_version, + ) + .cost_as_result() + .expect("expected to get cost"); assert_ne!(cost_with_cache, cost_without_cache); assert_eq!( diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index f32a1f358..ea207f1bb 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -502,9 +502,15 @@ mod tests { .expect("expected successful insertion 2"); assert_eq!( - Element::get(&merk, b"another-key", true, grove_version) - .unwrap() - .expect("expected successful get"), + Element::get( + &merk, + b"another-key", + true, + None:: String>, + grove_version + ) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value".to_vec()), ); } @@ -535,9 +541,15 @@ mod tests { assert!(!inserted); assert_eq!(previous, None); assert_eq!( - Element::get(&merk, b"another-key", true, grove_version) - .unwrap() - .expect("expected successful get"), + Element::get( + &merk, + b"another-key", + true, + None:: String>, + grove_version + ) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value".to_vec()), ); } @@ -580,9 +592,15 @@ mod tests { let merk = empty_path_merk_read_only(&*storage, grove_version); assert_eq!( - Element::get(&merk, b"another-key", true, grove_version) - .unwrap() - .expect("expected successful get"), + Element::get( + &merk, + b"another-key", + true, + None:: String>, + grove_version + ) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value2".to_vec()), ); } @@ -604,9 +622,15 @@ mod tests { assert_eq!(previous, None); assert_eq!( - Element::get(&merk, b"another-key", true, grove_version) - .unwrap() - .expect("expected successful get"), + Element::get( + &merk, + b"another-key", + true, + None:: String>, + grove_version + ) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value2".to_vec()), ); } diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index f14d3c735..66a617e7c 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -27,6 +27,7 @@ use grovedb_version::{ #[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; +use crate::operations::proof::util::path_as_slices_hex_to_ascii; #[cfg(any(feature = "full", feature = "verify"))] use crate::Element; #[cfg(feature = "full")] @@ -764,8 +765,14 @@ impl Element { subtree, grove_version, { - Element::get(&subtree, key, query_options.allow_cache, grove_version) - .unwrap_add_cost(&mut cost) + Element::get( + &subtree, + key, + query_options.allow_cache, + Some(|| format!("path is {}", path_as_slices_hex_to_ascii(path))), + grove_version, + ) + .unwrap_add_cost(&mut cost) } ); match element_res { diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index 88022d662..8381460c0 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -22,6 +22,9 @@ use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; +use crate::{ + operations::proof::util::path_as_slices_hex_to_ascii, replication::utils::path_to_string, +}; #[cfg(feature = "full")] use crate::{ reference_path::{path_from_reference_path_type, path_from_reference_qualified_path_type}, @@ -295,7 +298,7 @@ impl GroveDb { let merk_to_get_from = cost_return_on_error!( &mut cost, - self.open_transactional_merk_at_path(path, transaction, None, grove_version) + self.open_transactional_merk_at_path(path.clone(), transaction, None, grove_version) .map_err(|e| match e { Error::InvalidParentLayerPath(s) => { Error::PathParentLayerNotFound(s) @@ -304,7 +307,14 @@ impl GroveDb { }) ); - Element::get(&merk_to_get_from, key, allow_cache, grove_version).add_cost(cost) + Element::get( + &merk_to_get_from, + key, + allow_cache, + Some(|| format!("path is {}", path)), + grove_version, + ) + .add_cost(cost) } /// Get tree item without following references @@ -353,7 +363,7 @@ impl GroveDb { let merk_to_get_from = cost_return_on_error!( &mut cost, - self.open_non_transactional_merk_at_path(path, None, grove_version) + self.open_non_transactional_merk_at_path(path.clone(), None, grove_version) .map_err(|e| match e { Error::InvalidParentLayerPath(s) => { Error::PathParentLayerNotFound(s) @@ -362,7 +372,14 @@ impl GroveDb { }) ); - Element::get(&merk_to_get_from, key, allow_cache, grove_version).add_cost(cost) + Element::get( + &merk_to_get_from, + key, + allow_cache, + Some(|| format!("path is {}", path)), + grove_version, + ) + .add_cost(cost) } /// Get tree item without following references @@ -444,14 +461,26 @@ impl GroveDb { ) ); - Element::get(&merk_to_get_from, parent_key, true, grove_version) + Element::get( + &merk_to_get_from, + parent_key, + true, + Some(|| format!("path is {}", path)), + grove_version, + ) } else { let merk_to_get_from = cost_return_on_error!( &mut cost, self.open_non_transactional_merk_at_path(parent_path, None, grove_version) ); - Element::get(&merk_to_get_from, parent_key, true, grove_version) + Element::get( + &merk_to_get_from, + parent_key, + true, + Some(|| format!("path is {}", path)), + grove_version, + ) } .unwrap_add_cost(&mut cost); match element { diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index a7f01eb7b..08af3f5fb 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -3326,9 +3326,15 @@ mod tests { ) .unwrap() .expect("cannot open merk"); - let result_element = Element::get(&subtree, b"key3", true, grove_version) - .unwrap() - .unwrap(); + let result_element = Element::get( + &subtree, + b"key3", + true, + None:: String>, + grove_version, + ) + .unwrap() + .unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); } // Insert a new tree with transaction @@ -3375,9 +3381,15 @@ mod tests { ) .unwrap() .expect("cannot open merk"); - let result_element = Element::get(&subtree, b"key4", true, grove_version) - .unwrap() - .unwrap(); + let result_element = Element::get( + &subtree, + b"key4", + true, + None:: String>, + grove_version, + ) + .unwrap() + .unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); // Should be able to retrieve instances created before transaction @@ -3395,9 +3407,15 @@ mod tests { ) .unwrap() .expect("cannot open merk"); - let result_element = Element::get(&subtree, b"key3", true, grove_version) - .unwrap() - .unwrap(); + let result_element = Element::get( + &subtree, + b"key3", + true, + None:: String>, + grove_version, + ) + .unwrap() + .unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); } diff --git a/path/Cargo.toml b/path/Cargo.toml index 92417664f..8056e1b39 100644 --- a/path/Cargo.toml +++ b/path/Cargo.toml @@ -9,3 +9,4 @@ documentation = "https://docs.rs/grovedb-path" repository = "https://github.com/dashpay/grovedb" [dependencies] +hex = "0.4.3" diff --git a/path/src/subtree_path.rs b/path/src/subtree_path.rs index 437f911af..ae8cd9000 100644 --- a/path/src/subtree_path.rs +++ b/path/src/subtree_path.rs @@ -34,7 +34,10 @@ //! combined with it's various `From` implementations it can cover slices, owned //! subtree paths and other path references if use as generic [Into]. -use std::hash::{Hash, Hasher}; +use std::{ + fmt::{Display, Formatter}, + hash::{Hash, Hasher}, +}; use crate::{ subtree_path_builder::{SubtreePathBuilder, SubtreePathRelative}, @@ -48,6 +51,51 @@ pub struct SubtreePath<'b, B> { pub(crate) ref_variant: SubtreePathInner<'b, B>, } +fn hex_to_ascii(hex_value: &[u8]) -> String { + // Define the set of allowed characters + const ALLOWED_CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ\ + abcdefghijklmnopqrstuvwxyz\ + 0123456789_-/\\[]@"; + + // Check if all characters in hex_value are allowed + if hex_value.iter().all(|&c| ALLOWED_CHARS.contains(&c)) { + // Try to convert to UTF-8 + String::from_utf8(hex_value.to_vec()) + .unwrap_or_else(|_| format!("0x{}", hex::encode(hex_value))) + } else { + // Hex encode and prepend "0x" + format!("0x{}", hex::encode(hex_value)) + } +} + +impl<'b, B: AsRef<[u8]>> Display for SubtreePath<'b, B> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match &self.ref_variant { + SubtreePathInner::Slice(slice) => { + let ascii_path = slice + .iter() + .map(|e| hex_to_ascii(e.as_ref())) + .collect::>() + .join("/"); + write!(f, "{}", ascii_path) + } + SubtreePathInner::SubtreePath(subtree_path) => { + let ascii_path = subtree_path + .to_vec() + .into_iter() + .map(|a| hex_to_ascii(a.as_slice())) + .collect::>() + .join("/"); + write!(f, "{}", ascii_path) + } + SubtreePathInner::SubtreePathIter(iter) => { + let ascii_path = iter.clone().map(hex_to_ascii).collect::>().join("/"); + write!(f, "{}", ascii_path) + } + } + } +} + /// Wrapped inner representation of subtree path ref. #[derive(Debug)] pub(crate) enum SubtreePathInner<'b, B> { diff --git a/path/src/util/compact_bytes.rs b/path/src/util/compact_bytes.rs index 1e4362cb5..c44b6dd94 100644 --- a/path/src/util/compact_bytes.rs +++ b/path/src/util/compact_bytes.rs @@ -66,7 +66,7 @@ impl CompactBytes { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub(crate) struct CompactBytesIter<'a> { bytes: &'a CompactBytes, offset_back: usize, From 4375da5ae7745cf933fa2e10d0c36aae900c8c7f Mon Sep 17 00:00:00 2001 From: fominok Date: Tue, 14 Jan 2025 11:34:08 +0100 Subject: [PATCH 18/30] feat: append error context (#353) * fix compilation errors * add error context --- grovedb/src/debugger.rs | 63 ++++++++++++++++++++++++------- grovedb/src/element/get.rs | 55 +++++++-------------------- grovedb/src/element/insert.rs | 48 ++++++----------------- grovedb/src/element/query.rs | 13 +++---- grovedb/src/error.rs | 47 ++++++++++++++++++++++- grovedb/src/operations/get/mod.rs | 44 ++++++--------------- grovedb/src/tests/mod.rs | 36 +++++------------- grovedbg-types/src/lib.rs | 25 ++++++++++++ 8 files changed, 172 insertions(+), 159 deletions(-) diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 1920ff813..696a3a6ac 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -397,22 +397,28 @@ fn merk_proof_node_to_grovedbg(node: Node) -> Result { - let element = crate::Element::deserialize(&value, GroveVersion::latest())?; - MerkProofNode::KVValueHashFeatureType( - key, - element_to_grovedbg(element), - hash, - grovedbg_types::TreeFeatureType::BasicMerkNode, - ) - } - Node::KVValueHashFeatureType(key, value, hash, TreeFeatureType::SummedMerkNode(sum)) => { + Node::KVValueHashFeatureType(key, value, hash, feature_type) => { let element = crate::Element::deserialize(&value, GroveVersion::latest())?; + let node_feature_type = match feature_type { + TreeFeatureType::BasicMerkNode => grovedbg_types::TreeFeatureType::BasicMerkNode, + TreeFeatureType::SummedMerkNode(sum) => { + grovedbg_types::TreeFeatureType::SummedMerkNode(sum) + } + TreeFeatureType::BigSummedMerkNode(sum) => { + grovedbg_types::TreeFeatureType::BigSummedMerkNode(sum) + } + TreeFeatureType::CountedMerkNode(count) => { + grovedbg_types::TreeFeatureType::CountedMerkNode(count) + } + TreeFeatureType::CountedSummedMerkNode(count, sum) => { + grovedbg_types::TreeFeatureType::CountedSummedMerkNode(count, sum) + } + }; MerkProofNode::KVValueHashFeatureType( key, element_to_grovedbg(element), hash, - grovedbg_types::TreeFeatureType::SummedMerkNode(sum), + node_feature_type, ) } Node::KVRefValueHash(key, value, hash) => { @@ -597,6 +603,28 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { sum, element_flags, }, + crate::Element::BigSumTree(root_key, sum, element_flags) => { + grovedbg_types::Element::BigSumTree { + root_key, + sum, + element_flags, + } + } + crate::Element::CountTree(root_key, count, element_flags) => { + grovedbg_types::Element::CountTree { + root_key, + count, + element_flags, + } + } + crate::Element::CountSumTree(root_key, count, sum, element_flags) => { + grovedbg_types::Element::CountSumTree { + root_key, + count, + sum, + element_flags, + } + } } } @@ -628,8 +656,17 @@ fn node_to_update( right_merk_hash, feature_type: match feature_type { TreeFeatureType::BasicMerkNode => grovedbg_types::TreeFeatureType::BasicMerkNode, - TreeFeatureType::SummedMerkNode(x) => { - grovedbg_types::TreeFeatureType::SummedMerkNode(x) + TreeFeatureType::SummedMerkNode(sum) => { + grovedbg_types::TreeFeatureType::SummedMerkNode(sum) + } + TreeFeatureType::BigSummedMerkNode(sum) => { + grovedbg_types::TreeFeatureType::BigSummedMerkNode(sum) + } + TreeFeatureType::CountedMerkNode(count) => { + grovedbg_types::TreeFeatureType::CountedMerkNode(count) + } + TreeFeatureType::CountedSummedMerkNode(count, sum) => { + grovedbg_types::TreeFeatureType::CountedSummedMerkNode(count, sum) } }, value_hash, diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 02ed9f95b..e894545e9 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -32,7 +32,6 @@ impl Element { merk: &Merk, key: K, allow_cache: bool, - extra_error_info: Option String>, grove_version: &GroveVersion, ) -> CostResult { check_grovedb_v0_with_cost!("get", grove_version.grovedb_versions.element.get); @@ -44,19 +43,14 @@ impl Element { } else { String::new() }; - let extra_error_info_string = extra_error_info - .map(|callback| format!(" {}", callback())) - .unwrap_or(String::new()); Error::PathKeyNotFound(format!( - "get: key 0x{} {}not found in Merk that has a root key [{}] and is of type \ - {}{}", + "get: key 0x{} {}not found in Merk that has a root key [{}] and is of type {}", hex::encode(key), key_single_byte, merk.root_key() .map(hex::encode) .unwrap_or("None".to_string()), merk.merk_type, - extra_error_info_string, )) }) }) @@ -326,6 +320,8 @@ impl Element { allow_cache: bool, grove_version: &GroveVersion, ) -> CostResult { + use crate::error::GroveDbErrorExt; + check_grovedb_v0_with_cost!( "get_with_absolute_refs", grove_version @@ -337,13 +333,8 @@ impl Element { let element = cost_return_on_error!( &mut cost, - Self::get( - merk, - key.as_ref(), - allow_cache, - Some(|| { format!("path is {}", path_as_slices_hex_to_ascii(path)) }), - grove_version - ) + Self::get(merk, key.as_ref(), allow_cache, grove_version) + .add_context(format!("path is {}", path_as_slices_hex_to_ascii(path))) ); let absolute_element = cost_return_on_error_no_add!( @@ -435,15 +426,9 @@ mod tests { .unwrap(); assert_eq!( - Element::get( - &merk, - b"another-key", - true, - None:: String>, - grove_version - ) - .unwrap() - .expect("expected successful get"), + Element::get(&merk, b"another-key", true, grove_version) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value".to_vec()), ); @@ -453,24 +438,12 @@ mod tests { .unwrap() .expect("expected successful insertion 2"); - let cost_with_cache = Element::get( - &merk, - b"another-key", - true, - None:: String>, - grove_version, - ) - .cost_as_result() - .expect("expected to get cost"); - let cost_without_cache = Element::get( - &merk, - b"another-key", - false, - None:: String>, - grove_version, - ) - .cost_as_result() - .expect("expected to get cost"); + let cost_with_cache = Element::get(&merk, b"another-key", true, grove_version) + .cost_as_result() + .expect("expected to get cost"); + let cost_without_cache = Element::get(&merk, b"another-key", false, grove_version) + .cost_as_result() + .expect("expected to get cost"); assert_ne!(cost_with_cache, cost_without_cache); assert_eq!( diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index ea207f1bb..f32a1f358 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -502,15 +502,9 @@ mod tests { .expect("expected successful insertion 2"); assert_eq!( - Element::get( - &merk, - b"another-key", - true, - None:: String>, - grove_version - ) - .unwrap() - .expect("expected successful get"), + Element::get(&merk, b"another-key", true, grove_version) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value".to_vec()), ); } @@ -541,15 +535,9 @@ mod tests { assert!(!inserted); assert_eq!(previous, None); assert_eq!( - Element::get( - &merk, - b"another-key", - true, - None:: String>, - grove_version - ) - .unwrap() - .expect("expected successful get"), + Element::get(&merk, b"another-key", true, grove_version) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value".to_vec()), ); } @@ -592,15 +580,9 @@ mod tests { let merk = empty_path_merk_read_only(&*storage, grove_version); assert_eq!( - Element::get( - &merk, - b"another-key", - true, - None:: String>, - grove_version - ) - .unwrap() - .expect("expected successful get"), + Element::get(&merk, b"another-key", true, grove_version) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value2".to_vec()), ); } @@ -622,15 +604,9 @@ mod tests { assert_eq!(previous, None); assert_eq!( - Element::get( - &merk, - b"another-key", - true, - None:: String>, - grove_version - ) - .unwrap() - .expect("expected successful get"), + Element::get(&merk, b"another-key", true, grove_version) + .unwrap() + .expect("expected successful get"), Element::new_item(b"value2".to_vec()), ); } diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 66a617e7c..ef60edb79 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -744,6 +744,8 @@ impl Element { add_element_function: fn(PathQueryPushArgs, &GroveVersion) -> CostResult<(), Error>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { + use crate::error::GroveDbErrorExt; + check_grovedb_v0_with_cost!( "query_item", grove_version.grovedb_versions.element.query_item @@ -765,14 +767,9 @@ impl Element { subtree, grove_version, { - Element::get( - &subtree, - key, - query_options.allow_cache, - Some(|| format!("path is {}", path_as_slices_hex_to_ascii(path))), - grove_version, - ) - .unwrap_add_cost(&mut cost) + Element::get(&subtree, key, query_options.allow_cache, grove_version) + .add_context(format!("path is {}", path_as_slices_hex_to_ascii(path))) + .unwrap_add_cost(&mut cost) } ); match element_res { diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index 923439352..1d05ee41f 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -2,8 +2,9 @@ use std::convert::Infallible; +use grovedb_costs::CostResult; + /// GroveDB Errors -#[cfg(any(feature = "full", feature = "verify"))] #[derive(Debug, thiserror::Error)] pub enum Error { #[error("infallible")] @@ -158,6 +159,50 @@ pub enum Error { CyclicError(&'static str), } +impl Error { + pub fn add_context(&mut self, append: impl AsRef) { + match self { + Self::MissingReference(s) + | Self::InternalError(s) + | Self::InvalidProof(s) + | Self::PathKeyNotFound(s) + | Self::PathNotFound(s) + | Self::PathParentLayerNotFound(s) + | Self::CorruptedReferencePathKeyNotFound(s) + | Self::CorruptedReferencePathNotFound(s) + | Self::CorruptedReferencePathParentLayerNotFound(s) + | Self::InvalidParentLayerPath(s) + | Self::InvalidPath(s) + | Self::CorruptedPath(s) + | Self::CorruptedData(s) + | Self::CorruptedStorage(s) + | Self::DeleteUpTreeStopHeightMoreThanInitialPathSize(s) + | Self::JustInTimeElementFlagsClientError(s) + | Self::SplitRemovalBytesClientError(s) + | Self::ClientReturnedNonClientError(s) + | Self::PathNotFoundInCacheForEstimatedCosts(s) + | Self::NotSupported(s) => { + s.push_str(", "); + s.push_str(append.as_ref()); + } + _ => {} + } + } +} + +pub trait GroveDbErrorExt { + fn add_context(self, append: impl AsRef) -> Self; +} + +impl GroveDbErrorExt for CostResult { + fn add_context(self, append: impl AsRef) -> Self { + self.map_err(|mut e| { + e.add_context(append.as_ref()); + e + }) + } +} + impl From for Error { fn from(_value: Infallible) -> Self { Self::Infallible diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index 8381460c0..6c0bc3de7 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -22,9 +22,7 @@ use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; -use crate::{ - operations::proof::util::path_as_slices_hex_to_ascii, replication::utils::path_to_string, -}; +use crate::error::GroveDbErrorExt; #[cfg(feature = "full")] use crate::{ reference_path::{path_from_reference_path_type, path_from_reference_qualified_path_type}, @@ -307,14 +305,9 @@ impl GroveDb { }) ); - Element::get( - &merk_to_get_from, - key, - allow_cache, - Some(|| format!("path is {}", path)), - grove_version, - ) - .add_cost(cost) + Element::get(&merk_to_get_from, key, allow_cache, grove_version) + .add_context(format!("path is {}", path)) + .add_cost(cost) } /// Get tree item without following references @@ -372,14 +365,9 @@ impl GroveDb { }) ); - Element::get( - &merk_to_get_from, - key, - allow_cache, - Some(|| format!("path is {}", path)), - grove_version, - ) - .add_cost(cost) + Element::get(&merk_to_get_from, key, allow_cache, grove_version) + .add_context(format!("path is {}", path)) + .add_cost(cost) } /// Get tree item without following references @@ -461,26 +449,16 @@ impl GroveDb { ) ); - Element::get( - &merk_to_get_from, - parent_key, - true, - Some(|| format!("path is {}", path)), - grove_version, - ) + Element::get(&merk_to_get_from, parent_key, true, grove_version) + .add_context(format!("path is {}", path)) } else { let merk_to_get_from = cost_return_on_error!( &mut cost, self.open_non_transactional_merk_at_path(parent_path, None, grove_version) ); - Element::get( - &merk_to_get_from, - parent_key, - true, - Some(|| format!("path is {}", path)), - grove_version, - ) + Element::get(&merk_to_get_from, parent_key, true, grove_version) + .add_context(format!("path is {}", path)) } .unwrap_add_cost(&mut cost); match element { diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 08af3f5fb..a7f01eb7b 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -3326,15 +3326,9 @@ mod tests { ) .unwrap() .expect("cannot open merk"); - let result_element = Element::get( - &subtree, - b"key3", - true, - None:: String>, - grove_version, - ) - .unwrap() - .unwrap(); + let result_element = Element::get(&subtree, b"key3", true, grove_version) + .unwrap() + .unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); } // Insert a new tree with transaction @@ -3381,15 +3375,9 @@ mod tests { ) .unwrap() .expect("cannot open merk"); - let result_element = Element::get( - &subtree, - b"key4", - true, - None:: String>, - grove_version, - ) - .unwrap() - .unwrap(); + let result_element = Element::get(&subtree, b"key4", true, grove_version) + .unwrap() + .unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); // Should be able to retrieve instances created before transaction @@ -3407,15 +3395,9 @@ mod tests { ) .unwrap() .expect("cannot open merk"); - let result_element = Element::get( - &subtree, - b"key3", - true, - None:: String>, - grove_version, - ) - .unwrap() - .unwrap(); + let result_element = Element::get(&subtree, b"key3", true, grove_version) + .unwrap() + .unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); } diff --git a/grovedbg-types/src/lib.rs b/grovedbg-types/src/lib.rs index dd9fc0076..fb9c6d90f 100644 --- a/grovedbg-types/src/lib.rs +++ b/grovedbg-types/src/lib.rs @@ -125,6 +125,28 @@ pub enum Element { #[serde_as(as = "Option")] element_flags: Option>, }, + BigSumTree { + #[serde_as(as = "Option")] + root_key: Option, + sum: i128, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + CountTree { + #[serde_as(as = "Option")] + root_key: Option, + count: u64, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + CountSumTree { + #[serde_as(as = "Option")] + root_key: Option, + count: u64, + sum: i64, + #[serde_as(as = "Option")] + element_flags: Option>, + }, Item { #[serde_as(as = "Base64")] value: Vec, @@ -261,6 +283,9 @@ pub enum MerkProofNode { pub enum TreeFeatureType { BasicMerkNode, SummedMerkNode(i64), + BigSummedMerkNode(i128), + CountedMerkNode(u64), + CountedSummedMerkNode(u64, i64), } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] From 384894fb5ffc0e9e6b6c67f6736139b900c8194d Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 15 Jan 2025 11:43:10 +0100 Subject: [PATCH 19/30] merge fixes --- grovedb/src/batch/estimated_costs/average_case_costs.rs | 6 ------ grovedb/src/batch/estimated_costs/worst_case_costs.rs | 2 -- grovedb/src/element/get.rs | 4 +--- grovedb/src/element/mod.rs | 3 --- grovedb/src/operations/delete/mod.rs | 1 - grovedb/src/operations/delete/worst_case.rs | 2 +- merk/Cargo.toml | 2 ++ merk/src/test_utils/temp_merk.rs | 1 - merk/src/tree/kv.rs | 1 - merk/src/tree/link.rs | 2 +- merk/src/tree/mod.rs | 8 ++++---- merk/src/tree/tree_feature_type.rs | 3 ++- 12 files changed, 11 insertions(+), 24 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 1d133ce6b..a488447a1 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -10,12 +10,6 @@ use std::{ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::RootHashKeyAndSum; -#[cfg(feature = "minimal")] -use grovedb_merk::{ - estimated_costs::average_case_costs::{average_case_merk_propagate, EstimatedLayerInformation}, - IsSumTree -}; #[cfg(feature = "minimal")] use grovedb_merk::estimated_costs::average_case_costs::{ average_case_merk_propagate, EstimatedLayerInformation, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 84bfa4f82..c5ebc2d04 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -16,8 +16,6 @@ use grovedb_merk::estimated_costs::worst_case_costs::{ }; use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "minimal")] -use grovedb_merk::RootHashKeyAndSum; -#[cfg(feature = "minimal")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; #[cfg(feature = "minimal")] diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 3f8d062f8..6e21e3b91 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -5,7 +5,6 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::tree::kv::KV; #[cfg(feature = "minimal")] use grovedb_merk::Merk; #[cfg(feature = "minimal")] @@ -19,13 +18,12 @@ use grovedb_version::{ }; use integer_encoding::VarInt; -use crate::element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}; #[cfg(feature = "minimal")] use crate::{ element::{CostSize, SUM_ITEM_COST_SIZE}, operations::proof::util::path_as_slices_hex_to_ascii, }; -#[cfg(feature = "full")] +#[cfg(feature = "minimal")] use crate::{Element, Error, Hash}; impl Element { diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 020e44892..6dfbc3c8e 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -29,9 +29,6 @@ use grovedb_merk::estimated_costs::SUM_AND_COUNT_LAYER_COST_SIZE; #[cfg(any(feature = "minimal", feature = "verify"))] use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; #[cfg(feature = "minimal")] -use grovedb_merk::estimated_costs::{LAYER_COST_SIZE, SUM_LAYER_COST_SIZE}; -#[cfg(feature = "minimal")] -#[cfg(feature = "minimal")] use grovedb_merk::estimated_costs::{ BIG_SUM_LAYER_COST_SIZE, LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, }; diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index f4c781b63..8bf2e0ea5 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -18,7 +18,6 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{proofs::Query, KVIterator}; #[cfg(feature = "minimal")] use grovedb_merk::{proofs::Query, KVIterator, MaybeTree}; #[cfg(feature = "minimal")] diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 1d4ee1809..2c6bc8f4e 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -27,7 +27,7 @@ impl GroveDb { key: &KeyInfo, stop_path_height: Option, validate: bool, - intermediate_tree_info: IntMap<(TreeType, u32)>, + intermediate_tree_info: IntMap, max_element_size: u32, grove_version: &GroveVersion, ) -> CostResult, Error> { diff --git a/merk/Cargo.toml b/merk/Cargo.toml index a720f1273..ccc9f6e8d 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -24,6 +24,7 @@ integer-encoding = "4.0.0" thiserror = "2.0.11" serde = { version = "1.0.210", features = ["derive"], optional = true } rand = { version = "0.8.5", features = ["small_rng"], optional = true } +byteorder = { version = "1.5.0", optional = true } [dependencies.colored] version = "3.0.0" @@ -46,6 +47,7 @@ default = ["full"] proof_debug = [] serde = ["dep:serde", "indexmap/serde"] minimal = ["num_cpus", + "byteorder", "ed", "blake3", "grovedb-storage", diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 6eeaabdf6..54febc43f 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -40,7 +40,6 @@ use grovedb_storage::{ }; use grovedb_version::version::GroveVersion; -use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "minimal")] use crate::Merk; use crate::{merk::tree_type::TreeType, tree::kv::ValueDefinedCostType}; diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index 6db3f1055..8ad5349b3 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -12,7 +12,6 @@ use integer_encoding::VarInt; #[cfg(feature = "minimal")] use super::hash::{CryptoHash, HASH_LENGTH, NULL_HASH}; -use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "minimal")] use crate::{ merk::NodeType, diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index ef0976e01..5e45b6fd1 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -806,6 +806,6 @@ mod test { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 0, ]; let link = Link::decode(bytes.as_slice()).expect("expected to decode a link"); - assert_eq!(link.aggregateData(), AggregateData::NoAggregateData); + assert_eq!(link.aggregate_data(), AggregateData::NoAggregateData); } } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 12ddae6a4..0199bbbc4 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -456,7 +456,7 @@ impl TreeNode { ( // 36 = 32 Hash + 1 key length + 2 child heights + 1 feature type link.key().len() as u32 + 36, - match link.aggregateData() { + match link.aggregate_data() { AggregateData::NoAggregateData => 0, AggregateData::Sum(s) => s.encode_var_vec().len() as u32, AggregateData::BigSum(_) => 16 as u32, @@ -507,7 +507,7 @@ impl TreeNode { #[inline] pub fn child_aggregate_sum_data_as_i64(&self, left: bool) -> Result { match self.link(left) { - Some(link) => match link.aggregateData() { + Some(link) => match link.aggregate_data() { AggregateData::NoAggregateData => Ok(0), AggregateData::Sum(s) => Ok(s), AggregateData::BigSum(_) => Err(Error::BigSumTreeUnderNormalSumTree( @@ -525,7 +525,7 @@ impl TreeNode { #[inline] pub fn child_aggregate_count_data_as_u64(&self, left: bool) -> Result { match self.link(left) { - Some(link) => match link.aggregateData() { + Some(link) => match link.aggregate_data() { AggregateData::NoAggregateData => Ok(0), AggregateData::Sum(_) => Ok(0), AggregateData::BigSum(_) => Ok(0), @@ -541,7 +541,7 @@ impl TreeNode { #[inline] pub fn child_aggregate_sum_data_as_i128(&self, left: bool) -> i128 { match self.link(left) { - Some(link) => match link.aggregateData() { + Some(link) => match link.aggregate_data() { AggregateData::NoAggregateData => 0, AggregateData::Sum(s) => s as i128, AggregateData::BigSum(s) => s, diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 008978285..2ab787630 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -3,7 +3,7 @@ #[cfg(any(feature = "minimal", feature = "verify"))] use std::io::{Read, Write}; -#[cfg(feature = "minimal")] +#[cfg(any(feature = "minimal", feature = "verify"))] use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "minimal")] use ed::Terminated; @@ -159,6 +159,7 @@ impl TreeFeatureType { #[cfg(feature = "minimal")] impl Terminated for TreeFeatureType {} +#[cfg(feature = "minimal")] impl Encode for TreeFeatureType { #[inline] fn encode_into(&self, dest: &mut W) -> ed::Result<()> { From 20705a37801c029358f863e0199a5229bc5952b8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 09:37:33 +0700 Subject: [PATCH 20/30] fixes --- .../estimated_costs/average_case_costs.rs | 4 ++-- grovedb/src/element/constructor.rs | 18 +++++++++--------- grovedb/src/element/get.rs | 3 +-- grovedb/src/element/helpers.rs | 1 - grovedb/src/element/query.rs | 1 - grovedb/src/lib.rs | 1 - merk/src/estimated_costs/average_case_costs.rs | 4 ++-- merk/src/estimated_costs/mod.rs | 2 +- merk/src/merk/mod.rs | 2 +- merk/src/tree/tree_feature_type.rs | 2 +- 10 files changed, 17 insertions(+), 21 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 6909b7b6d..379e1e88e 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -11,8 +11,8 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; #[cfg(feature = "minimal")] -use grovedb_merk::{ - estimated_costs::average_case_costs::{average_case_merk_propagate, EstimatedLayerInformation}, +use grovedb_merk::estimated_costs::average_case_costs::{ + average_case_merk_propagate, EstimatedLayerInformation, }; use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "minimal")] diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index d8e88fb82..a6bf00bb5 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -170,13 +170,13 @@ impl Element { Element::SumTree(maybe_root_key, sum_value, flags) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a big sum tree without flags pub fn new_big_sum_tree(maybe_root_key: Option>) -> Self { Element::BigSumTree(maybe_root_key, 0, None) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a big sum tree with flags pub fn new_big_sum_tree_with_flags( maybe_root_key: Option>, @@ -185,7 +185,7 @@ impl Element { Element::BigSumTree(maybe_root_key, 0, flags) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a big sum tree with flags and sum value pub fn new_big_sum_tree_with_flags_and_sum_value( maybe_root_key: Option>, @@ -195,13 +195,13 @@ impl Element { Element::BigSumTree(maybe_root_key, big_sum_value, flags) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a count tree without flags pub fn new_count_tree(maybe_root_key: Option>) -> Self { Element::CountTree(maybe_root_key, 0, None) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a count tree with flags pub fn new_count_tree_with_flags( maybe_root_key: Option>, @@ -210,7 +210,7 @@ impl Element { Element::CountTree(maybe_root_key, 0, flags) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a count tree with flags and sum value pub fn new_count_tree_with_flags_and_count_value( maybe_root_key: Option>, @@ -220,13 +220,13 @@ impl Element { Element::CountTree(maybe_root_key, count_value, flags) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a count sum tree without flags pub fn new_count_sum_tree(maybe_root_key: Option>) -> Self { Element::CountSumTree(maybe_root_key, 0, 0, None) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a count sum tree with flags pub fn new_count_sum_tree_with_flags( maybe_root_key: Option>, @@ -235,7 +235,7 @@ impl Element { Element::CountSumTree(maybe_root_key, 0, 0, flags) } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Set element to a count sum tree with flags and sum value pub fn new_count_sum_tree_with_flags_and_sum_and_count_value( maybe_root_key: Option>, diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index ecfcb1522..694010fce 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -5,7 +5,6 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; - #[cfg(feature = "minimal")] use grovedb_merk::Merk; #[cfg(feature = "minimal")] @@ -148,7 +147,7 @@ impl Element { } } - #[cfg(feature = "full")] + #[cfg(feature = "minimal")] /// Get an element directly from storage under a key /// Merk does not need to be loaded fn get_optional_from_storage_v0<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 1b2e20842..fb0da0556 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -6,7 +6,6 @@ use grovedb_merk::tree::kv::{ ValueDefinedCostType, ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, }; - #[cfg(feature = "minimal")] use grovedb_merk::{ merk::{tree_type::TreeType, NodeType}, diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 51def5258..d0022c909 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -8,7 +8,6 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostContext, CostResult, CostsExt, OperationCost, }; - #[cfg(feature = "minimal")] use grovedb_merk::merk::tree_type::TreeType; #[cfg(feature = "minimal")] diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 8ccf5e6c3..3e8e28b0b 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -186,7 +186,6 @@ use grovedb_merk::{ tree::{combine_hash, value_hash}, BatchEntry, CryptoHash, KVIterator, Merk, }; - #[cfg(any(feature = "minimal", feature = "verify"))] pub use grovedb_merk::{ merk::tree_type::{MaybeTree, TreeType}, diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 7ae281049..2ad358cad 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -502,7 +502,7 @@ pub fn add_average_case_merk_propagate( )), } } -#[cfg(feature = "full")] +#[cfg(feature = "minimal")] /// Add average case cost for propagating a merk fn add_average_case_merk_propagate_v1( cost: &mut OperationCost, @@ -773,7 +773,7 @@ fn add_average_case_merk_propagate_v1( Ok(()) } -#[cfg(feature = "full")] +#[cfg(feature = "minimal")] /// Add average case cost for propagating a merk fn add_average_case_merk_propagate_v0( cost: &mut OperationCost, diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index 2073c3247..3f02c90e0 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -29,7 +29,7 @@ pub const LAYER_COST_SIZE: u32 = 3; /// The cost of a sum value pub const SUM_VALUE_EXTRA_COST: u32 = 9; -#[cfg(any(feature = "full", feature = "verify"))] +#[cfg(any(feature = "minimal", feature = "verify"))] /// The cost of a count value pub const COUNT_VALUE_EXTRA_COST: u32 = 9; diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index b2ae3cdec..efd906311 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -243,7 +243,7 @@ impl MerkType { } } -#[cfg(any(feature = "full", feature = "verify"))] +#[cfg(any(feature = "minimal", feature = "verify"))] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum NodeType { NormalNode, diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 2ab787630..286058423 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -49,7 +49,7 @@ impl TreeFeatureType { } } -#[cfg(any(feature = "full", feature = "verify"))] +#[cfg(any(feature = "minimal", feature = "verify"))] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum AggregateData { NoAggregateData, From 28ceab568e98a6c3d521a6e19b5d148331a63940 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:06:53 +0700 Subject: [PATCH 21/30] fixes --- costs/Cargo.toml | 2 +- grovedb-epoch-based-storage-flags/Cargo.toml | 4 ++-- grovedb-version/Cargo.toml | 2 +- grovedb/Cargo.toml | 18 +++++++++--------- grovedbg-types/Cargo.toml | 2 +- merk/Cargo.toml | 15 +++++++-------- merk/src/tree/tree_feature_type.rs | 9 +++------ node-grove/Cargo.toml | 4 ++-- path/Cargo.toml | 2 +- storage/Cargo.toml | 8 ++++---- visualize/Cargo.toml | 2 +- 11 files changed, 32 insertions(+), 36 deletions(-) diff --git a/costs/Cargo.toml b/costs/Cargo.toml index 382404d69..a98dc02dc 100644 --- a/costs/Cargo.toml +++ b/costs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-costs" -version = "2.2.1" +version = "3.0.0" edition = "2021" license = "MIT" description = "Costs extension crate for GroveDB" diff --git a/grovedb-epoch-based-storage-flags/Cargo.toml b/grovedb-epoch-based-storage-flags/Cargo.toml index 2e32ec2b2..83df14633 100644 --- a/grovedb-epoch-based-storage-flags/Cargo.toml +++ b/grovedb-epoch-based-storage-flags/Cargo.toml @@ -2,13 +2,13 @@ name = "grovedb-epoch-based-storage-flags" authors = ["Samuel Westrich "] description = "Epoch based storage flags for GroveDB" -version = "2.2.1" +version = "3.0.0" edition = "2021" license = "MIT" repository = "https://github.com/dashpay/grovedb" [dependencies] -grovedb-costs = { version = "2.2.1", path = "../costs" } +grovedb-costs = { version = "3.0.0", path = "../costs" } hex = { version = "0.4.3" } integer-encoding = { version = "4.0.0" } diff --git a/grovedb-version/Cargo.toml b/grovedb-version/Cargo.toml index 5cbd2dbec..d1e6b08ba 100644 --- a/grovedb-version/Cargo.toml +++ b/grovedb-version/Cargo.toml @@ -2,7 +2,7 @@ name = "grovedb-version" authors = ["Samuel Westrich "] description = "Versioning library for Platform" -version = "2.2.1" +version = "3.0.0" edition = "2021" license = "MIT" repository = "https://github.com/dashpay/grovedb" diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 66dd2c6cb..5a2d998e6 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb" description = "Fully featured database using balanced hierarchical authenticated data structures" -version = "2.2.1" +version = "3.0.0" authors = ["Samuel Westrich ", "Wisdom Ogwu "] edition = "2021" license = "MIT" @@ -11,13 +11,13 @@ readme = "../README.md" documentation = "https://docs.rs/grovedb" [dependencies] -grovedb-costs = { version = "2.2.1", path = "../costs" , optional = true } -grovedbg-types = { version = "2.2.1", path = "../grovedbg-types", optional = true } -grovedb-merk = { version = "2.2.1", path = "../merk", optional = true, default-features = false } -grovedb-path = { version = "2.2.1", path = "../path" } -grovedb-storage = { version = "2.2.1", path = "../storage", optional = true } -grovedb-version = { version = "2.2.1", path = "../grovedb-version" } -grovedb-visualize = { version = "2.2.1", path = "../visualize", optional = true } +grovedb-costs = { version = "3.0.0", path = "../costs" , optional = true } +grovedbg-types = { version = "3.0.0", path = "../grovedbg-types", optional = true } +grovedb-merk = { version = "3.0.0", path = "../merk", optional = true, default-features = false } +grovedb-path = { version = "3.0.0", path = "../path" } +grovedb-storage = { version = "3.0.0", path = "../storage", optional = true } +grovedb-version = { version = "3.0.0", path = "../grovedb-version" } +grovedb-visualize = { version = "3.0.0", path = "../visualize", optional = true } axum = { version = "=0.7.5", features = ["macros"], optional = true } bincode = { version = "2.0.0-rc.3" } @@ -36,7 +36,7 @@ zip-extensions = { version = "0.8.1", optional = true } serde = { version = "1.0.210", features = ["derive"], optional = true } [dev-dependencies] -grovedb-epoch-based-storage-flags = { version = "2.2.1", path = "../grovedb-epoch-based-storage-flags" } +grovedb-epoch-based-storage-flags = { version = "3.0.0", path = "../grovedb-epoch-based-storage-flags" } criterion = "0.5.1" hex = "0.4.3" diff --git a/grovedbg-types/Cargo.toml b/grovedbg-types/Cargo.toml index 7c5eb549d..357ce04ab 100644 --- a/grovedbg-types/Cargo.toml +++ b/grovedbg-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedbg-types" -version = "2.2.1" +version = "3.0.0" edition = "2021" description = "Common type definitions for data exchange over GroveDBG protocol" authors = ["Evgeny Fomin "] diff --git a/merk/Cargo.toml b/merk/Cargo.toml index ea80685f4..07d0b930e 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb-merk" description = "Merkle key/value store adapted for GroveDB" -version = "2.2.1" +version = "3.0.0" authors = ["Samuel Westrich ", "Wisdom Ogwu ", "Matt Bell "] edition = "2021" license = "MIT" @@ -11,11 +11,11 @@ readme = "README.md" documentation = "https://docs.rs/grovedb-merk" [dependencies] -grovedb-costs = { version = "2.2.1" , path = "../costs" } -grovedb-path = { version = "2.2.1", path = "../path" } -grovedb-storage = { version = "2.2.1", path = "../storage", optional = true } -grovedb-version = { version = "2.2.1", path = "../grovedb-version" } -grovedb-visualize = { version = "2.2.1", path = "../visualize" } +grovedb-costs = { version = "3.0.0" , path = "../costs" } +grovedb-path = { version = "3.0.0", path = "../path" } +grovedb-storage = { version = "3.0.0", path = "../storage", optional = true } +grovedb-version = { version = "3.0.0", path = "../grovedb-version" } +grovedb-visualize = { version = "3.0.0", path = "../visualize" } bincode = { version = "2.0.0-rc.3" } hex = "0.4.3" @@ -24,7 +24,7 @@ integer-encoding = "4.0.0" thiserror = "2.0.11" serde = { version = "1.0.210", features = ["derive"], optional = true } rand = { version = "0.8.5", features = ["small_rng"], optional = true } -byteorder = { version = "1.5.0", optional = true } +byteorder = { version = "1.5.0" } [dependencies.colored] version = "3.0.0" @@ -47,7 +47,6 @@ default = ["full"] proof_debug = [] serde = ["dep:serde", "indexmap/serde"] minimal = ["num_cpus", - "byteorder", "ed", "blake3", "grovedb-storage", diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 286058423..60d9a149b 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -15,11 +15,9 @@ use grovedb_costs::TreeCostType; use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(any(feature = "minimal", feature = "verify"))] -use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; -use crate::{ - merk::{tree_type::TreeType, NodeType}, - TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, -}; +use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode, BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}; +#[cfg(feature = "minimal")] +use crate::merk::{tree_type::TreeType, NodeType}; #[cfg(any(feature = "minimal", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -159,7 +157,6 @@ impl TreeFeatureType { #[cfg(feature = "minimal")] impl Terminated for TreeFeatureType {} -#[cfg(feature = "minimal")] impl Encode for TreeFeatureType { #[inline] fn encode_into(&self, dest: &mut W) -> ed::Result<()> { diff --git a/node-grove/Cargo.toml b/node-grove/Cargo.toml index bd91146b7..3654bd5b2 100644 --- a/node-grove/Cargo.toml +++ b/node-grove/Cargo.toml @@ -10,8 +10,8 @@ exclude = ["index.node"] crate-type = ["cdylib"] [dependencies] -grovedb = { version = "2.2.1", path = "../grovedb", features = ["full", "estimated_costs"] } -grovedb-version = { version = "2.2.1", path = "../grovedb-version" } +grovedb = { version = "3.0.0", path = "../grovedb", features = ["full", "estimated_costs"] } +grovedb-version = { version = "3.0.0", path = "../grovedb-version" } [dependencies.neon] version = "0.10.1" diff --git a/path/Cargo.toml b/path/Cargo.toml index 2fd13f568..91738bfe5 100644 --- a/path/Cargo.toml +++ b/path/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-path" -version = "2.2.1" +version = "3.0.0" edition = "2021" license = "MIT" description = "Path extension crate for GroveDB" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index e2db3bb6b..409ddaede 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-storage" -version = "2.2.1" +version = "3.0.0" edition = "2021" license = "MIT" description = "Storage extension crate for GroveDB" @@ -9,9 +9,9 @@ documentation = "https://docs.rs/grovedb-storage" repository = "https://github.com/dashpay/grovedb" [dependencies] -grovedb-costs = { version = "2.2.1", path = "../costs" } -grovedb-path = { version = "2.2.1", path = "../path" } -grovedb-visualize = { version = "2.2.1", path = "../visualize" } +grovedb-costs = { version = "3.0.0", path = "../costs" } +grovedb-path = { version = "3.0.0", path = "../path" } +grovedb-visualize = { version = "3.0.0", path = "../visualize" } blake3 = { version = "1.5.1", optional = true } hex = "0.4.3" diff --git a/visualize/Cargo.toml b/visualize/Cargo.toml index 233341a25..60b09efde 100644 --- a/visualize/Cargo.toml +++ b/visualize/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-visualize" -version = "2.2.1" +version = "3.0.0" edition = "2021" license = "MIT" description = "Debug prints extension crate for GroveDB" From fcfee9ebf6598fab7cb732724f11f22110902c0d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:17:01 +0700 Subject: [PATCH 22/30] fixes --- merk/src/tree/tree_feature_type.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 60d9a149b..82a448c29 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -35,6 +35,7 @@ pub enum TreeFeatureType { CountedSummedMerkNode(u64, i64), } +#[cfg(feature = "minimal")] impl TreeFeatureType { pub fn node_type(&self) -> NodeType { match self { @@ -47,7 +48,7 @@ impl TreeFeatureType { } } -#[cfg(any(feature = "minimal", feature = "verify"))] +#[cfg(feature = "minimal")] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum AggregateData { NoAggregateData, @@ -57,6 +58,7 @@ pub enum AggregateData { CountAndSum(u64, i64), } +#[cfg(feature = "minimal")] impl AggregateData { pub fn parent_tree_type(&self) -> TreeType { match self { @@ -106,6 +108,7 @@ impl AggregateData { } } +#[cfg(feature = "minimal")] impl From for AggregateData { fn from(value: TreeFeatureType) -> Self { match value { @@ -157,6 +160,7 @@ impl TreeFeatureType { #[cfg(feature = "minimal")] impl Terminated for TreeFeatureType {} +#[cfg(any(feature = "minimal", feature = "verify"))] impl Encode for TreeFeatureType { #[inline] fn encode_into(&self, dest: &mut W) -> ed::Result<()> { From 7760e0435ce28d71de5d842fa89b5bdf71d71b5c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:18:40 +0700 Subject: [PATCH 23/30] fixes --- merk/src/tree/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 0199bbbc4..460edbce4 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -59,7 +59,7 @@ use kv::KV; pub use link::Link; #[cfg(feature = "minimal")] pub use ops::{AuxMerkBatch, BatchEntry, MerkBatch, Op, PanicSource}; -#[cfg(any(feature = "minimal", feature = "verify"))] +#[cfg(feature = "minimal")] pub use tree_feature_type::AggregateData; #[cfg(any(feature = "minimal", feature = "verify"))] pub use tree_feature_type::TreeFeatureType; From 867b476221994972d041826f147d0daaa82777ac Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:18:56 +0700 Subject: [PATCH 24/30] fixes --- merk/src/estimated_costs/average_case_costs.rs | 2 +- merk/src/estimated_costs/mod.rs | 1 - merk/src/tree/tree_feature_type.rs | 6 ++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 2ad358cad..60cfd578e 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -4,10 +4,10 @@ use grovedb_costs::{CostResult, CostsExt, OperationCost}; #[cfg(feature = "minimal")] use grovedb_version::{check_grovedb_v0_or_v1, error::GroveVersionError, version::GroveVersion}; -use crate::merk::{tree_type::TreeType, NodeType}; #[cfg(feature = "minimal")] use integer_encoding::VarInt; +use crate::merk::{tree_type::TreeType, NodeType}; #[cfg(feature = "minimal")] use crate::{ error::Error, diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index 3f02c90e0..a0ee94c44 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -8,7 +8,6 @@ use integer_encoding::VarInt; #[cfg(feature = "minimal")] use crate::merk::{tree_type::TreeType, NodeType}; #[cfg(feature = "minimal")] - use crate::{tree::kv::KV, HASH_BLOCK_SIZE_U32, HASH_LENGTH_U32}; #[cfg(feature = "minimal")] diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 82a448c29..e82248e73 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -14,10 +14,12 @@ use grovedb_costs::TreeCostType; #[cfg(any(feature = "minimal", feature = "verify"))] use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; -#[cfg(any(feature = "minimal", feature = "verify"))] -use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode, BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}; #[cfg(feature = "minimal")] use crate::merk::{tree_type::TreeType, NodeType}; +#[cfg(any(feature = "minimal", feature = "verify"))] +use crate::tree::tree_feature_type::TreeFeatureType::{ + BasicMerkNode, BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode, SummedMerkNode, +}; #[cfg(any(feature = "minimal", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] From 4005e34f6e677e8a8ef5730410a7a367838ca4d0 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:26:46 +0700 Subject: [PATCH 25/30] more refactoring --- .../src/batch/estimated_costs/average_case_costs.rs | 4 ++-- grovedb/src/batch/estimated_costs/worst_case_costs.rs | 2 +- grovedb/src/batch/just_in_time_reference_update.rs | 2 +- grovedb/src/batch/mod.rs | 2 +- grovedb/src/batch/single_deletion_cost_tests.rs | 2 +- .../src/batch/single_sum_item_deletion_cost_tests.rs | 2 +- grovedb/src/element/delete.rs | 2 +- grovedb/src/element/get.rs | 2 +- grovedb/src/element/helpers.rs | 4 +++- grovedb/src/element/mod.rs | 8 +++++--- grovedb/src/element/query.rs | 4 ++-- grovedb/src/estimated_costs/average_case_costs.rs | 5 ++--- grovedb/src/estimated_costs/worst_case_costs.rs | 5 +++-- grovedb/src/lib.rs | 9 ++++----- grovedb/src/operations/delete/average_case.rs | 2 +- grovedb/src/operations/delete/worst_case.rs | 4 ++-- grovedb/src/operations/get/average_case.rs | 2 +- grovedb/src/operations/get/worst_case.rs | 2 +- grovedb/src/operations/is_empty_tree.rs | 2 +- grovedb/src/replication.rs | 4 ++-- grovedb/src/replication/state_sync_session.rs | 2 +- merk/src/estimated_costs/average_case_costs.rs | 2 +- merk/src/estimated_costs/mod.rs | 4 +++- merk/src/lib.rs | 8 ++++++-- merk/src/merk/mod.rs | 10 +++------- merk/src/merk/open.rs | 4 ++-- merk/src/merk/restore.rs | 6 ++++-- merk/src/merk/source.rs | 2 +- merk/src/test_utils/mod.rs | 2 +- merk/src/test_utils/temp_merk.rs | 2 +- merk/src/tree/tree_feature_type.rs | 4 +++- merk/src/{merk => }/tree_type.rs | 0 32 files changed, 62 insertions(+), 53 deletions(-) rename merk/src/{merk => }/tree_type.rs (100%) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 379e1e88e..310c58639 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -14,7 +14,7 @@ use grovedb_costs::{ use grovedb_merk::estimated_costs::average_case_costs::{ average_case_merk_propagate, EstimatedLayerInformation, }; -use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; +use grovedb_merk::{tree::AggregateData, tree_type::TreeType, RootHashKeyAndAggregateData}; #[cfg(feature = "minimal")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; @@ -303,7 +303,7 @@ mod tests { EstimatedLayerSizes::{AllItems, AllSubtrees}, EstimatedSumTrees::{NoSumTrees, SomeSumTrees}, }, - merk::tree_type::TreeType, + tree_type::TreeType, }; use grovedb_version::version::GroveVersion; diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index c5ebc2d04..b48109ade 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -14,7 +14,7 @@ use grovedb_costs::{ use grovedb_merk::estimated_costs::worst_case_costs::{ worst_case_merk_propagate, WorstCaseLayerInformation, }; -use grovedb_merk::{merk::tree_type::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; +use grovedb_merk::{tree::AggregateData, tree_type::TreeType, RootHashKeyAndAggregateData}; #[cfg(feature = "minimal")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index 26242b400..06081eb25 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -9,8 +9,8 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ - merk::tree_type::TreeType, tree::{kv::KV, value_hash, TreeNode}, + tree_type::TreeType, CryptoHash, Merk, }; use grovedb_storage::StorageContext; diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 36a692d2b..ee3b1ad74 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -48,11 +48,11 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ - merk::tree_type::TreeType, tree::{ kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, value_hash, AggregateData, NULL_HASH, }, + tree_type::TreeType, CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndAggregateData, }; use grovedb_path::SubtreePath; diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index e7e59318e..6e784e571 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -7,7 +7,7 @@ mod tests { Identifier, StorageRemovalPerEpochByIdentifier, StorageRemovedBytes::SectionedStorageRemoval, }; - use grovedb_merk::merk::tree_type::TreeType; + use grovedb_merk::tree_type::TreeType; use grovedb_version::version::GroveVersion; use intmap::IntMap; diff --git a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs index 0c10519b6..4763af162 100644 --- a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs @@ -2,7 +2,7 @@ #[cfg(feature = "minimal")] mod tests { - use grovedb_merk::merk::tree_type::TreeType; + use grovedb_merk::tree_type::TreeType; use grovedb_version::version::GroveVersion; use crate::{ diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index d81dc2ac1..17095d72f 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -6,7 +6,7 @@ use grovedb_costs::OperationCost; #[cfg(feature = "minimal")] use grovedb_costs::{storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt}; #[cfg(feature = "minimal")] -use grovedb_merk::merk::tree_type::TreeType; +use grovedb_merk::tree_type::TreeType; #[cfg(feature = "minimal")] use grovedb_merk::{BatchEntry, Error as MerkError, Merk, MerkOptions, Op}; #[cfg(feature = "minimal")] diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 694010fce..40868c77a 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -379,7 +379,7 @@ impl Element { #[cfg(feature = "minimal")] #[cfg(test)] mod tests { - use grovedb_merk::merk::tree_type::TreeType; + use grovedb_merk::tree_type::TreeType; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage, StorageBatch}; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index fb0da0556..0a958eaf2 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -7,8 +7,10 @@ use grovedb_merk::tree::kv::{ ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, }; #[cfg(feature = "minimal")] +use grovedb_merk::tree_type::TreeType; +#[cfg(feature = "minimal")] use grovedb_merk::{ - merk::{tree_type::TreeType, NodeType}, + merk::NodeType, MaybeTree, TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, }; diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 0f1fedf99..bfac686a9 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -24,16 +24,16 @@ mod serialize; use std::fmt; use bincode::{Decode, Encode}; -#[cfg(any(feature = "minimal", feature = "verify"))] +#[cfg(feature = "minimal")] use grovedb_merk::estimated_costs::SUM_AND_COUNT_LAYER_COST_SIZE; -#[cfg(any(feature = "minimal", feature = "verify"))] +#[cfg(feature = "minimal")] use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; #[cfg(feature = "minimal")] use grovedb_merk::estimated_costs::{ BIG_SUM_LAYER_COST_SIZE, LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, }; #[cfg(feature = "minimal")] -use grovedb_merk::merk::tree_type::TreeType; +use grovedb_merk::tree_type::TreeType; #[cfg(feature = "minimal")] use grovedb_visualize::visualize_to_vec; @@ -90,10 +90,12 @@ pub type BigSumValue = i128; /// int 64 count value pub type CountValue = u64; +#[cfg(feature = "minimal")] pub trait CostSize { fn cost_size(&self) -> u32; } +#[cfg(feature = "minimal")] impl CostSize for TreeType { fn cost_size(&self) -> u32 { match self { diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index d0022c909..68e570562 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -9,14 +9,14 @@ use grovedb_costs::{ OperationCost, }; #[cfg(feature = "minimal")] -use grovedb_merk::merk::tree_type::TreeType; -#[cfg(feature = "minimal")] use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(feature = "minimal")] use grovedb_merk::proofs::query::SubqueryBranch; #[cfg(feature = "minimal")] use grovedb_merk::proofs::Query; #[cfg(feature = "minimal")] +use grovedb_merk::tree_type::TreeType; +#[cfg(feature = "minimal")] use grovedb_path::SubtreePath; #[cfg(feature = "minimal")] use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, StorageContext}; diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 21b1ea035..e779d9b83 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -12,8 +12,8 @@ use grovedb_merk::{ add_average_case_merk_replace_layered, EstimatedLayerInformation, }, }, - merk::tree_type::TreeType, tree::TreeNode, + tree_type::TreeType, HASH_LENGTH, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; @@ -616,8 +616,7 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ estimated_costs::average_case_costs::add_average_case_get_merk_node, - merk::tree_type::TreeType, test_utils::make_batch_seq, tree::kv::ValueDefinedCostType, - Merk, + test_utils::make_batch_seq, tree::kv::ValueDefinedCostType, tree_type::TreeType, Merk, }; use grovedb_storage::{ rocksdb_storage::RocksDbStorage, worst_case_costs::WorstKeyLength, Storage, StorageBatch, diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index 681169512..c6f889f97 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -14,8 +14,8 @@ use grovedb_merk::{ MERK_BIGGEST_VALUE_SIZE, }, }, - merk::tree_type::TreeType, tree::TreeNode, + tree_type::TreeType, HASH_LENGTH, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; @@ -497,9 +497,10 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ estimated_costs::worst_case_costs::add_worst_case_get_merk_node, - merk::{tree_type::TreeType, NodeType}, + merk::NodeType, test_utils::{empty_path_merk, empty_path_merk_read_only, make_batch_seq}, tree::kv::ValueDefinedCostType, + tree_type::TreeType, }; use grovedb_storage::{ rocksdb_storage::{test_utils::TempStorage, RocksDbStorage}, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 3e8e28b0b..5f36729bd 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -180,17 +180,16 @@ pub use grovedb_merk::proofs::query::query_item::QueryItem; pub use grovedb_merk::proofs::Query; #[cfg(feature = "minimal")] use grovedb_merk::tree::kv::ValueDefinedCostType; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use grovedb_merk::tree::AggregateData; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use grovedb_merk::tree_type::{MaybeTree, TreeType}; #[cfg(feature = "minimal")] use grovedb_merk::{ self, tree::{combine_hash, value_hash}, BatchEntry, CryptoHash, KVIterator, Merk, }; -#[cfg(any(feature = "minimal", feature = "verify"))] -pub use grovedb_merk::{ - merk::tree_type::{MaybeTree, TreeType}, - tree::AggregateData, -}; #[cfg(feature = "minimal")] use grovedb_path::SubtreePath; #[cfg(feature = "minimal")] diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 0f9ade1f7..6e8b0158d 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -8,7 +8,7 @@ use grovedb_merk::{ average_case_costs::EstimatedLayerInformation, worst_case_costs::add_average_case_cost_for_is_empty_tree_except, }, - merk::tree_type::TreeType, + tree_type::TreeType, HASH_LENGTH_U32, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 2c6bc8f4e..894541491 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -4,8 +4,8 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ - estimated_costs::worst_case_costs::add_worst_case_cost_for_is_empty_tree_except, - merk::tree_type::TreeType, tree::kv::KV, + estimated_costs::worst_case_costs::add_worst_case_cost_for_is_empty_tree_except, tree::kv::KV, + tree_type::TreeType, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; use grovedb_version::{ diff --git a/grovedb/src/operations/get/average_case.rs b/grovedb/src/operations/get/average_case.rs index 82f17ff27..d920ad011 100644 --- a/grovedb/src/operations/get/average_case.rs +++ b/grovedb/src/operations/get/average_case.rs @@ -3,7 +3,7 @@ #[cfg(feature = "minimal")] use grovedb_costs::OperationCost; #[cfg(feature = "minimal")] -use grovedb_merk::merk::tree_type::TreeType; +use grovedb_merk::tree_type::TreeType; #[cfg(feature = "minimal")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; diff --git a/grovedb/src/operations/get/worst_case.rs b/grovedb/src/operations/get/worst_case.rs index c374bb574..e6382dd81 100644 --- a/grovedb/src/operations/get/worst_case.rs +++ b/grovedb/src/operations/get/worst_case.rs @@ -3,7 +3,7 @@ #[cfg(feature = "minimal")] use grovedb_costs::OperationCost; #[cfg(feature = "minimal")] -use grovedb_merk::merk::tree_type::TreeType; +use grovedb_merk::tree_type::TreeType; #[cfg(feature = "minimal")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 2f68f5a24..4dec3abf3 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -2,7 +2,7 @@ #[cfg(feature = "minimal")] use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; -use grovedb_merk::merk::tree_type::TreeType; +use grovedb_merk::tree_type::TreeType; use grovedb_path::SubtreePath; #[cfg(feature = "minimal")] use grovedb_version::error::GroveVersionError; diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 7eab6e0e0..000f97b27 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -2,7 +2,7 @@ mod state_sync_session; use std::pin::Pin; -use grovedb_merk::{merk::tree_type::TreeType, tree::hash::CryptoHash, ChunkProducer}; +use grovedb_merk::{tree::hash::CryptoHash, tree_type::TreeType, ChunkProducer}; use grovedb_path::SubtreePath; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -256,8 +256,8 @@ impl GroveDb { pub(crate) mod utils { use grovedb_merk::{ ed::Encode, - merk::tree_type::TreeType, proofs::{Decoder, Op}, + tree_type::TreeType, }; use crate::{replication::ChunkIdentifier, Error}; diff --git a/grovedb/src/replication/state_sync_session.rs b/grovedb/src/replication/state_sync_session.rs index 678bc0b3e..a8831f073 100644 --- a/grovedb/src/replication/state_sync_session.rs +++ b/grovedb/src/replication/state_sync_session.rs @@ -6,8 +6,8 @@ use std::{ }; use grovedb_merk::{ - merk::tree_type::TreeType, tree::{kv::ValueDefinedCostType, value_hash}, + tree_type::TreeType, CryptoHash, Restorer, }; use grovedb_path::SubtreePath; diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 60cfd578e..3b535767a 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -7,7 +7,6 @@ use grovedb_version::{check_grovedb_v0_or_v1, error::GroveVersionError, version: #[cfg(feature = "minimal")] use integer_encoding::VarInt; -use crate::merk::{tree_type::TreeType, NodeType}; #[cfg(feature = "minimal")] use crate::{ error::Error, @@ -15,6 +14,7 @@ use crate::{ tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, HASH_LENGTH_U32, }; +use crate::{merk::NodeType, tree_type::TreeType}; #[cfg(feature = "minimal")] /// Average key size diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index a0ee94c44..0ef4d18c0 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -6,7 +6,9 @@ use grovedb_costs::OperationCost; use integer_encoding::VarInt; #[cfg(feature = "minimal")] -use crate::merk::{tree_type::TreeType, NodeType}; +use crate::merk::NodeType; +#[cfg(feature = "minimal")] +use crate::tree_type::TreeType; #[cfg(feature = "minimal")] use crate::{tree::kv::KV, HASH_BLOCK_SIZE_U32, HASH_LENGTH_U32}; diff --git a/merk/src/lib.rs b/merk/src/lib.rs index e41ed7392..51d15afb0 100644 --- a/merk/src/lib.rs +++ b/merk/src/lib.rs @@ -65,6 +65,8 @@ pub mod error; #[cfg(any(feature = "minimal", feature = "verify"))] pub mod estimated_costs; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub mod tree_type; #[cfg(feature = "minimal")] mod visualize; @@ -79,13 +81,15 @@ pub use tree::{ }; #[cfg(any(feature = "minimal", feature = "verify"))] pub use tree::{CryptoHash, TreeFeatureType}; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use tree_type::MaybeTree; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use tree_type::TreeType; #[cfg(feature = "minimal")] pub use crate::merk::{ defaults::ROOT_KEY_KEY, prove::{ProofConstructionResult, ProofWithoutEncodingResult}, - tree_type::MaybeTree, - tree_type::TreeType, KVIterator, Merk, MerkType, RootHashKeyAndAggregateData, }; #[cfg(feature = "minimal")] diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index efd906311..27d679c12 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -41,7 +41,6 @@ pub mod open; pub mod prove; pub mod restore; pub mod source; -pub mod tree_type; use std::{ cell::Cell, @@ -58,7 +57,6 @@ use grovedb_costs::{ use grovedb_storage::{self, Batch, RawIterator, StorageContext}; use grovedb_version::version::GroveVersion; use source::MerkSource; -use tree_type::TreeType; use crate::{ error::Error, @@ -75,6 +73,7 @@ use crate::{ kv::ValueDefinedCostType, AggregateData, AuxMerkBatch, CryptoHash, Op, RefWalker, TreeNode, NULL_HASH, }, + tree_type::TreeType, Error::{CostsError, EdError, StorageError}, Link, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, @@ -804,11 +803,8 @@ mod test { use super::{Merk, RefWalker}; use crate::{ - merk::{source::MerkSource, tree_type::TreeType}, - test_utils::*, - tree::kv::ValueDefinedCostType, - Op, - TreeFeatureType::BasicMerkNode, + merk::source::MerkSource, test_utils::*, tree::kv::ValueDefinedCostType, + tree_type::TreeType, Op, TreeFeatureType::BasicMerkNode, }; // TODO: Close and then reopen test diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index 302e9e890..a3d4c16e8 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -5,8 +5,8 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ - merk::tree_type::TreeType, tree::kv::ValueDefinedCostType, + tree_type::TreeType, Error, Merk, MerkType, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, }; @@ -103,7 +103,7 @@ mod test { use tempfile::TempDir; use crate::{ - merk::tree_type::TreeType, tree::kv::ValueDefinedCostType, Merk, Op, + tree::kv::ValueDefinedCostType, tree_type::TreeType, Merk, Op, TreeFeatureType::BasicMerkNode, }; diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index b5a75e375..0c1784fd9 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -36,7 +36,7 @@ use grovedb_version::version::GroveVersion; use crate::{ merk, - merk::{tree_type::TreeType, MerkSource}, + merk::MerkSource, proofs::{ chunk::{ chunk::{LEFT, RIGHT}, @@ -48,6 +48,7 @@ use crate::{ Node, Op, }, tree::{combine_hash, kv::ValueDefinedCostType, RefWalker, TreeNode}, + tree_type::TreeType, CryptoHash, Error, Error::{CostsError, StorageError}, Link, Merk, @@ -561,11 +562,12 @@ mod tests { use super::*; use crate::{ - merk::{chunks::ChunkProducer, tree_type::TreeType}, + merk::chunks::ChunkProducer, proofs::chunk::{ chunk::tests::traverse_get_node_hash, error::ChunkError::InvalidChunkProof, }, test_utils::{make_batch_seq, TempMerk}, + tree_type::TreeType, Error::ChunkRestoringError, Merk, PanicSource, }; diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs index a4f96027e..7c7568be8 100644 --- a/merk/src/merk/source.rs +++ b/merk/src/merk/source.rs @@ -3,8 +3,8 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ - merk::tree_type::TreeType, tree::{kv::ValueDefinedCostType, Fetch, TreeNode}, + tree_type::TreeType, Error, Link, Merk, }; diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index bdfa3219c..76eec9480 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -40,11 +40,11 @@ use rand::prelude::*; pub use temp_merk::TempMerk; use crate::{ - merk::tree_type::TreeType, tree::{ kv::{ValueDefinedCostType, KV}, BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, TreeNode, Walker, }, + tree_type::TreeType, Merk, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 54febc43f..a9b3b26e3 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -42,7 +42,7 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "minimal")] use crate::Merk; -use crate::{merk::tree_type::TreeType, tree::kv::ValueDefinedCostType}; +use crate::{tree::kv::ValueDefinedCostType, tree_type::TreeType}; #[cfg(feature = "minimal")] /// Wraps a Merk instance and deletes it from disk it once it goes out of scope. diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index e82248e73..579e50324 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -15,11 +15,13 @@ use grovedb_costs::TreeCostType; use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(feature = "minimal")] -use crate::merk::{tree_type::TreeType, NodeType}; +use crate::merk::NodeType; #[cfg(any(feature = "minimal", feature = "verify"))] use crate::tree::tree_feature_type::TreeFeatureType::{ BasicMerkNode, BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode, SummedMerkNode, }; +#[cfg(feature = "minimal")] +use crate::tree_type::TreeType; #[cfg(any(feature = "minimal", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] diff --git a/merk/src/merk/tree_type.rs b/merk/src/tree_type.rs similarity index 100% rename from merk/src/merk/tree_type.rs rename to merk/src/tree_type.rs From d36c2e81ae9a1a25414c3d3ef80ea779be8e9ff8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:32:19 +0700 Subject: [PATCH 26/30] more refactoring --- merk/src/tree_type.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/merk/src/tree_type.rs b/merk/src/tree_type.rs index ef845f21a..aeff62d60 100644 --- a/merk/src/tree_type.rs +++ b/merk/src/tree_type.rs @@ -1,6 +1,9 @@ use std::fmt; -use crate::{merk::NodeType, Error, TreeFeatureType}; +#[cfg(feature = "minimal")] +use crate::merk::NodeType; + +use crate::{Error, TreeFeatureType}; #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum MaybeTree { @@ -56,6 +59,7 @@ impl TreeType { } } + #[cfg(feature = "minimal")] pub const fn inner_node_type(&self) -> NodeType { match self { TreeType::NormalTree => NodeType::NormalNode, From 2f14efb70233144c4dd1204288c5e44a60dc2bbc Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:32:24 +0700 Subject: [PATCH 27/30] more refactoring --- merk/src/tree_type.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/merk/src/tree_type.rs b/merk/src/tree_type.rs index aeff62d60..6432f232a 100644 --- a/merk/src/tree_type.rs +++ b/merk/src/tree_type.rs @@ -2,7 +2,6 @@ use std::fmt; #[cfg(feature = "minimal")] use crate::merk::NodeType; - use crate::{Error, TreeFeatureType}; #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] From 7dbf36792e3bf691a68bfbd9866984250aabb23f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:41:47 +0700 Subject: [PATCH 28/30] more refactoring --- grovedb/src/element/helpers.rs | 39 ++++++++++++++++------------------ grovedb/src/lib.rs | 4 ++-- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 0a958eaf2..d5cf03219 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -1,24 +1,23 @@ //! Helpers //! Implements helper functions in Element -#[cfg(feature = "minimal")] -use grovedb_merk::tree::kv::{ - ValueDefinedCostType, - ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, -}; -#[cfg(feature = "minimal")] -use grovedb_merk::tree_type::TreeType; +#[cfg(any(feature = "minimal", feature = "verify"))] +use grovedb_merk::tree_type::{MaybeTree, TreeType}; #[cfg(feature = "minimal")] use grovedb_merk::{ merk::NodeType, - MaybeTree, - TreeFeatureType::{BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode}, -}; -#[cfg(feature = "minimal")] -use grovedb_merk::{ - tree::{kv::KV, TreeNode}, + tree::{ + kv::{ + ValueDefinedCostType, + ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, + KV, + }, + TreeNode, + }, TreeFeatureType, - TreeFeatureType::{BasicMerkNode, SummedMerkNode}, + TreeFeatureType::{ + BasicMerkNode, BigSummedMerkNode, CountedMerkNode, CountedSummedMerkNode, SummedMerkNode, + }, }; #[cfg(feature = "minimal")] use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -26,18 +25,16 @@ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::Grove use integer_encoding::VarInt; #[cfg(feature = "minimal")] -use crate::element::{BIG_SUM_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE}; +use crate::element::{ + BIG_SUM_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, + SUM_TREE_COST_SIZE, TREE_COST_SIZE, +}; #[cfg(feature = "minimal")] use crate::reference_path::path_from_reference_path_type; #[cfg(any(feature = "minimal", feature = "verify"))] use crate::reference_path::ReferencePathType; -#[cfg(feature = "minimal")] -use crate::{ - element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, - ElementFlags, -}; #[cfg(any(feature = "minimal", feature = "verify"))] -use crate::{Element, Error}; +use crate::{Element, ElementFlags, Error}; impl Element { #[cfg(any(feature = "minimal", feature = "verify"))] diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 5f36729bd..7b4e9c003 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -161,7 +161,7 @@ use std::{collections::HashMap, option::Option::None, path::Path}; use debugger::start_visualizer; #[cfg(any(feature = "minimal", feature = "verify"))] pub use element::Element; -#[cfg(feature = "minimal")] +#[cfg(any(feature = "minimal", feature = "verify"))] pub use element::ElementFlags; #[cfg(feature = "minimal")] use grovedb_costs::{ @@ -180,7 +180,7 @@ pub use grovedb_merk::proofs::query::query_item::QueryItem; pub use grovedb_merk::proofs::Query; #[cfg(feature = "minimal")] use grovedb_merk::tree::kv::ValueDefinedCostType; -#[cfg(any(feature = "minimal", feature = "verify"))] +#[cfg(feature = "minimal")] pub use grovedb_merk::tree::AggregateData; #[cfg(any(feature = "minimal", feature = "verify"))] pub use grovedb_merk::tree_type::{MaybeTree, TreeType}; From d8ae2d95f56381b4d104d3983b2f11ae3a968dc7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 16 Jan 2025 11:44:58 +0700 Subject: [PATCH 29/30] more refactoring --- grovedb/src/element/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index bfac686a9..069788978 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -55,7 +55,7 @@ pub type MaxReferenceHop = Option; #[cfg(feature = "minimal")] /// The cost of a tree pub const TREE_COST_SIZE: u32 = LAYER_COST_SIZE; // 3 -#[cfg(any(feature = "minimal", feature = "verify"))] +#[cfg(feature = "minimal")] /// The cost of a sum item /// /// It is 11 because we have 9 bytes for the sum value From b7e0cb2b8a40d15003bf63dc97d6999ec683aafc Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 16 Jan 2025 09:09:36 +0100 Subject: [PATCH 30/30] fix --- grovedb/src/batch/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index ee3b1ad74..3d35525c6 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1841,11 +1841,11 @@ impl GroveDb { GroveOp::ReplaceTreeRootKey { hash, root_key, - aggregate_data, + aggregate_data: aggregate_data_entry, } => { *hash = root_hash; *root_key = calculated_root_key; - *aggregate_data = *aggregate_data; + *aggregate_data_entry = aggregate_data; } GroveOp::InsertTreeWithRootHash { .. } => { return Err(Error::CorruptedCodeExecution(