From e76a10473841a4714e6373a81109b2d1a4dffdef Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 14 Feb 2024 16:09:46 +0000 Subject: [PATCH 01/24] . --- src/storable.rs | 126 ++++++++++++++++++++++++++++++++++++++++++ src/storable/tests.rs | 37 +++++++++++++ 2 files changed, 163 insertions(+) diff --git a/src/storable.rs b/src/storable.rs index dfcb55e8..efa8dc22 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -492,6 +492,132 @@ where }; } +impl Storable for (A, B, C) +where + A: Storable, + B: Storable, + C: Storable, +{ + fn to_bytes(&self) -> Cow<[u8]> { + match Self::BOUND { + Bound::Bounded { max_size, .. } => { + let mut bytes = vec![0; max_size as usize]; + let a_bytes = self.0.to_bytes(); + let b_bytes = self.1.to_bytes(); + let c_bytes = self.2.to_bytes(); + + let a_bounds = bounds::(); + let b_bounds = bounds::(); + let c_bounds = bounds::(); + + let a_max_size = a_bounds.max_size as usize; + let b_max_size = b_bounds.max_size as usize; + let c_max_size = c_bounds.max_size as usize; + + debug_assert!(a_bytes.len() <= a_max_size); + debug_assert!(b_bytes.len() <= b_max_size); + debug_assert!(c_bytes.len() <= c_max_size); + + bytes[0..a_bytes.len()].copy_from_slice(a_bytes.borrow()); + bytes[a_max_size..a_max_size + b_bytes.len()].copy_from_slice(b_bytes.borrow()); + bytes[a_max_size + b_max_size..a_max_size + b_max_size + c_bytes.len()] + .copy_from_slice(c_bytes.borrow()); + + let a_size_len = bytes_to_store_size(&a_bounds) as usize; + let b_size_len = bytes_to_store_size(&b_bounds) as usize; + let c_size_len = bytes_to_store_size(&c_bounds) as usize; + + let sizes_offset: usize = a_max_size + b_max_size + c_max_size; + + encode_size( + &mut bytes[sizes_offset..sizes_offset + a_size_len], + a_bytes.len(), + &a_bounds, + ); + encode_size( + &mut bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], + b_bytes.len(), + &b_bounds, + ); + encode_size( + &mut bytes[sizes_offset + a_size_len + b_size_len + ..sizes_offset + a_size_len + b_size_len + c_size_len], + c_bytes.len(), + &c_bounds, + ); + + Cow::Owned(bytes) + } + _ => todo!("Serializing tuples with unbounded types is not yet supported."), + } + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + match Self::BOUND { + Bound::Bounded { max_size, .. } => { + assert_eq!(bytes.len(), max_size as usize); + + let a_bounds = bounds::(); + let b_bounds = bounds::(); + let c_bounds = bounds::(); + let a_max_size = a_bounds.max_size as usize; + let b_max_size = b_bounds.max_size as usize; + let c_max_size = c_bounds.max_size as usize; + let sizes_offset = a_max_size + b_max_size + c_max_size; + + let a_size_len = bytes_to_store_size(&a_bounds) as usize; + let b_size_len = bytes_to_store_size(&b_bounds) as usize; + let c_size_len = bytes_to_store_size(&c_bounds) as usize; + let a_len = decode_size(&bytes[sizes_offset..sizes_offset + a_size_len], &a_bounds); + let b_len = decode_size( + &bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], + &b_bounds, + ); + let c_len = decode_size( + &bytes[sizes_offset + a_size_len + b_size_len + ..sizes_offset + a_size_len + b_size_len + c_size_len], + &c_bounds, + ); + + let a = A::from_bytes(Cow::Borrowed(&bytes[0..a_len])); + let b = B::from_bytes(Cow::Borrowed(&bytes[a_max_size..a_max_size + b_len])); + let c = C::from_bytes(Cow::Borrowed( + &bytes[a_max_size + b_max_size..a_max_size + b_max_size + c_len], + )); + + (a, b, c) + } + _ => todo!("Deserializing tuples with unbounded types is not yet supported."), + } + } + + const BOUND: Bound = { + match (A::BOUND, B::BOUND, C::BOUND) { + (Bound::Bounded { .. }, Bound::Bounded { .. }, Bound::Bounded { .. }) => { + let a_bounds = bounds::(); + let b_bounds = bounds::(); + let c_bounds = bounds::(); + + let max_size = a_bounds.max_size + + b_bounds.max_size + + c_bounds.max_size + + bytes_to_store_size(&a_bounds) + + bytes_to_store_size(&b_bounds) + + bytes_to_store_size(&c_bounds); + + let is_fixed_size = + a_bounds.is_fixed_size && b_bounds.is_fixed_size && c_bounds.is_fixed_size; + + Bound::Bounded { + max_size, + is_fixed_size, + } + } + _ => Bound::Unbounded, + } + }; +} + impl Storable for Option { fn to_bytes(&self) -> Cow<[u8]> { match self { diff --git a/src/storable/tests.rs b/src/storable/tests.rs index e04c55e5..7746688c 100644 --- a/src/storable/tests.rs +++ b/src/storable/tests.rs @@ -13,6 +13,15 @@ proptest! { prop_assert_eq!(tuple, Storable::from_bytes(bytes)); } + #[test] + fn tuple_with_three_elements_roundtrip(x in any::(), y in uniform20(any::()), z in uniform20(any::())) { + let tuple = (x, y, z); + let bytes = tuple.to_bytes(); + prop_assert_eq!(bytes.len(), 48); + prop_assert_eq!(tuple, Storable::from_bytes(bytes)); + } + + #[test] fn tuple_variable_width_u8_roundtrip(x in any::(), v in pvec(any::(), 0..40)) { let bytes = Blob::<48>::try_from(&v[..]).unwrap(); @@ -20,6 +29,14 @@ proptest! { prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); } + #[test] + fn tuple_with_three_elements_variable_width_u8_roundtrip(x in any::(), v1 in pvec(any::(), 0..40), v2 in pvec(any::(), 0..80)) { + let v1_bytes = Blob::<40>::try_from(&v1[..]).unwrap(); + let v2_bytes = Blob::<80>::try_from(&v2[..]).unwrap(); + let tuple = (x, v1_bytes, v2_bytes); + prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + #[test] fn tuple_variable_width_u16_roundtrip(x in any::(), v in pvec(any::(), 0..40)) { let bytes = Blob::<300>::try_from(&v[..]).unwrap(); @@ -27,6 +44,15 @@ proptest! { prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); } + #[test] + fn tuple_with_three_elements_variable_width_u16_roundtrip(x in any::(), v1 in pvec(any::(), 0..40), v2 in pvec(any::(), 0..80)) { + let v1_bytes = Blob::<300>::try_from(&v1[..]).unwrap(); + let v2_bytes = Blob::<300>::try_from(&v2[..]).unwrap(); + + let tuple = (x, v1_bytes, v2_bytes); + prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + #[test] fn f64_roundtrip(v in any::()) { prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); @@ -52,12 +78,23 @@ proptest! { prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); } + #[test] + fn optional_tuple_with_three_elements_roundtrip(v in proptest::option::of((any::(), uniform20(any::()), uniform20(any::())))) { + prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); + } + #[test] fn optional_tuple_variable_width_u8_roundtrip(v in proptest::option::of((any::(), pvec(any::(), 0..40)))) { let v = v.map(|(n, bytes)| (n, Blob::<48>::try_from(&bytes[..]).unwrap())); prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); } + #[test] + fn optional_tuple_with_three_elements_variable_width_u8_roundtrip(v in proptest::option::of((any::(), pvec(any::(), 0..40), pvec(any::(), 0..80)))) { + let v = v.map(|(n, bytes_1, bytes_2)| (n, Blob::<40>::try_from(&bytes_1[..]).unwrap(), Blob::<80>::try_from(&bytes_2[..]).unwrap())); + prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); + } + #[test] fn principal_roundtrip(mut bytes in pvec(any::(), 0..=28), tag in proptest::prop_oneof![Just(1),Just(2),Just(3),Just(4),Just(7)]) { bytes.push(tag); From b5c41f280658056432a75bd1fdee2698e8f4e200 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 14 Feb 2024 16:17:45 +0000 Subject: [PATCH 02/24] refactore bound calculation --- src/storable.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index efa8dc22..f11cffbe 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -594,23 +594,14 @@ where const BOUND: Bound = { match (A::BOUND, B::BOUND, C::BOUND) { (Bound::Bounded { .. }, Bound::Bounded { .. }, Bound::Bounded { .. }) => { - let a_bounds = bounds::(); - let b_bounds = bounds::(); + let a_b_bounds = bounds::<(A, B)>(); let c_bounds = bounds::(); - let max_size = a_bounds.max_size - + b_bounds.max_size - + c_bounds.max_size - + bytes_to_store_size(&a_bounds) - + bytes_to_store_size(&b_bounds) - + bytes_to_store_size(&c_bounds); - - let is_fixed_size = - a_bounds.is_fixed_size && b_bounds.is_fixed_size && c_bounds.is_fixed_size; - Bound::Bounded { - max_size, - is_fixed_size, + max_size: a_b_bounds.max_size + + c_bounds.max_size + + bytes_to_store_size(&c_bounds), + is_fixed_size: a_b_bounds.is_fixed_size && c_bounds.is_fixed_size, } } _ => Bound::Unbounded, From 5d15dae597da06307cc1a7524429f93460f9de21 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 15 Feb 2024 10:07:03 +0000 Subject: [PATCH 03/24] . --- src/storable.rs | 81 ++++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index f11cffbe..3049d9bf 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -498,6 +498,34 @@ where B: Storable, C: Storable, { + /*fn to_bytes(&self) -> Cow<[u8]> { + match Self::BOUND { + Bound::Bounded { max_size, .. } => { + let mut bytes = vec![0; max_size as usize]; + let a_bytes = self.0.to_bytes(); + let a_bounds = bounds::(); + let a_max_size = a_bounds.max_size as usize; + debug_assert!(a_bytes.len() <= a_max_size); + + bytes[0..a_bytes.len()].copy_from_slice(a_bytes.borrow()); + let a_size_len = bytes_to_store_size(&a_bounds) as usize; + encode_size( + &mut bytes[a_max_size..a_max_size + a_size_len], + a_bytes.len(), + &a_bounds, + ); + + let b_c_bytes = <(B, C)>::to_bytes(&(self.1, self.2)); + + bytes[a_max_size + a_size_len..a_max_size + a_size_len + b_c_bytes.len()] + .copy_from_slice(b_c_bytes.into_owned().as_slice()); + + Cow::Owned(bytes) + } + _ => todo!("Serializing tuples with unbounded types is not yet supported."), + } + }*/ + fn to_bytes(&self) -> Cow<[u8]> { match Self::BOUND { Bound::Bounded { max_size, .. } => { @@ -517,23 +545,24 @@ where debug_assert!(a_bytes.len() <= a_max_size); debug_assert!(b_bytes.len() <= b_max_size); debug_assert!(c_bytes.len() <= c_max_size); - - bytes[0..a_bytes.len()].copy_from_slice(a_bytes.borrow()); - bytes[a_max_size..a_max_size + b_bytes.len()].copy_from_slice(b_bytes.borrow()); - bytes[a_max_size + b_max_size..a_max_size + b_max_size + c_bytes.len()] - .copy_from_slice(c_bytes.borrow()); - let a_size_len = bytes_to_store_size(&a_bounds) as usize; let b_size_len = bytes_to_store_size(&b_bounds) as usize; let c_size_len = bytes_to_store_size(&c_bounds) as usize; - let sizes_offset: usize = a_max_size + b_max_size + c_max_size; + bytes[0..a_bytes.len()].copy_from_slice(a_bytes.borrow()); + encode_size( &mut bytes[sizes_offset..sizes_offset + a_size_len], a_bytes.len(), &a_bounds, ); + bytes[a_max_size + a_size_len..a_max_size + a_size_len + b_bytes.len()] + .copy_from_slice(b_bytes.borrow()); + bytes[a_max_size + a_size_len + b_max_size + ..a_max_size + a_size_len + b_max_size + c_bytes.len()] + .copy_from_slice(c_bytes.borrow()); + encode_size( &mut bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], b_bytes.len(), @@ -556,33 +585,15 @@ where match Self::BOUND { Bound::Bounded { max_size, .. } => { assert_eq!(bytes.len(), max_size as usize); - let a_bounds = bounds::(); - let b_bounds = bounds::(); - let c_bounds = bounds::(); let a_max_size = a_bounds.max_size as usize; - let b_max_size = b_bounds.max_size as usize; - let c_max_size = c_bounds.max_size as usize; - let sizes_offset = a_max_size + b_max_size + c_max_size; - let a_size_len = bytes_to_store_size(&a_bounds) as usize; - let b_size_len = bytes_to_store_size(&b_bounds) as usize; - let c_size_len = bytes_to_store_size(&c_bounds) as usize; - let a_len = decode_size(&bytes[sizes_offset..sizes_offset + a_size_len], &a_bounds); - let b_len = decode_size( - &bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], - &b_bounds, - ); - let c_len = decode_size( - &bytes[sizes_offset + a_size_len + b_size_len - ..sizes_offset + a_size_len + b_size_len + c_size_len], - &c_bounds, - ); + let a_len = decode_size(&bytes[a_max_size..a_max_size + a_size_len], &a_bounds); let a = A::from_bytes(Cow::Borrowed(&bytes[0..a_len])); - let b = B::from_bytes(Cow::Borrowed(&bytes[a_max_size..a_max_size + b_len])); - let c = C::from_bytes(Cow::Borrowed( - &bytes[a_max_size + b_max_size..a_max_size + b_max_size + c_len], + + let (b, c) = <(B, C)>::from_bytes(Cow::Borrowed( + &bytes[a_max_size + a_size_len..bytes.len()], )); (a, b, c) @@ -594,14 +605,14 @@ where const BOUND: Bound = { match (A::BOUND, B::BOUND, C::BOUND) { (Bound::Bounded { .. }, Bound::Bounded { .. }, Bound::Bounded { .. }) => { - let a_b_bounds = bounds::<(A, B)>(); - let c_bounds = bounds::(); + let a_bounds = bounds::(); + let b_c_bounds = bounds::<(B, C)>(); Bound::Bounded { - max_size: a_b_bounds.max_size - + c_bounds.max_size - + bytes_to_store_size(&c_bounds), - is_fixed_size: a_b_bounds.is_fixed_size && c_bounds.is_fixed_size, + max_size: a_bounds.max_size + + bytes_to_store_size(&a_bounds) + + b_c_bounds.max_size, + is_fixed_size: a_bounds.is_fixed_size && b_c_bounds.is_fixed_size, } } _ => Bound::Unbounded, From 93e0ef7b1255c4d9871fbe5470d9c592be36da8e Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 15 Feb 2024 10:31:41 +0000 Subject: [PATCH 04/24] . --- src/storable.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 3049d9bf..cd50ba43 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -502,6 +502,7 @@ where match Self::BOUND { Bound::Bounded { max_size, .. } => { let mut bytes = vec![0; max_size as usize]; + let a_bytes = self.0.to_bytes(); let a_bounds = bounds::(); let a_max_size = a_bounds.max_size as usize; @@ -515,10 +516,10 @@ where &a_bounds, ); - let b_c_bytes = <(B, C)>::to_bytes(&(self.1, self.2)); + let b_c_bytes = (self.1, self.2).to_bytes(); bytes[a_max_size + a_size_len..a_max_size + a_size_len + b_c_bytes.len()] - .copy_from_slice(b_c_bytes.into_owned().as_slice()); + .copy_from_slice(b_c_bytes.borrow()); Cow::Owned(bytes) } From 0dcdb57f7e2a4248d8d42ebf27da8019e3c0d763 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 15 Feb 2024 13:38:30 +0000 Subject: [PATCH 05/24] . --- src/storable.rs | 122 ++++++++++++++++-------------------------------- 1 file changed, 40 insertions(+), 82 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index cd50ba43..a068d416 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -492,90 +492,54 @@ where }; } +fn serialize_with_size(entry: &T, bytes: &mut Vec, start: usize) -> usize +where + T: Storable, +{ + let entry_bytes = entry.to_bytes(); + bytes[start..start + entry_bytes.len()].copy_from_slice(entry_bytes.borrow()); + let bounds = bounds::(); + let max_size = bounds.max_size as usize; + debug_assert!(entry_bytes.len() <= max_size); + let size_len = bytes_to_store_size(&bounds) as usize; + encode_size( + &mut bytes[start + max_size..start + max_size + size_len], + entry_bytes.len(), + &bounds, + ); + start + max_size + size_len +} + +fn deserialize_with_size(bytes: &Cow<[u8]>, start: usize) -> (T, usize) +where + T: Storable, +{ + let bounds = bounds::(); + let max_size = bounds.max_size as usize; + let size_len = bytes_to_store_size(&bounds) as usize; + let len = decode_size( + &bytes[start + max_size..start + max_size + size_len], + &bounds, + ); + + let a = T::from_bytes(Cow::Borrowed(&bytes[start..start + len])); + (a, start + max_size + size_len) +} + impl Storable for (A, B, C) where A: Storable, B: Storable, C: Storable, { - /*fn to_bytes(&self) -> Cow<[u8]> { - match Self::BOUND { - Bound::Bounded { max_size, .. } => { - let mut bytes = vec![0; max_size as usize]; - - let a_bytes = self.0.to_bytes(); - let a_bounds = bounds::(); - let a_max_size = a_bounds.max_size as usize; - debug_assert!(a_bytes.len() <= a_max_size); - - bytes[0..a_bytes.len()].copy_from_slice(a_bytes.borrow()); - let a_size_len = bytes_to_store_size(&a_bounds) as usize; - encode_size( - &mut bytes[a_max_size..a_max_size + a_size_len], - a_bytes.len(), - &a_bounds, - ); - - let b_c_bytes = (self.1, self.2).to_bytes(); - - bytes[a_max_size + a_size_len..a_max_size + a_size_len + b_c_bytes.len()] - .copy_from_slice(b_c_bytes.borrow()); - - Cow::Owned(bytes) - } - _ => todo!("Serializing tuples with unbounded types is not yet supported."), - } - }*/ - fn to_bytes(&self) -> Cow<[u8]> { match Self::BOUND { Bound::Bounded { max_size, .. } => { let mut bytes = vec![0; max_size as usize]; - let a_bytes = self.0.to_bytes(); - let b_bytes = self.1.to_bytes(); - let c_bytes = self.2.to_bytes(); - - let a_bounds = bounds::(); - let b_bounds = bounds::(); - let c_bounds = bounds::(); - - let a_max_size = a_bounds.max_size as usize; - let b_max_size = b_bounds.max_size as usize; - let c_max_size = c_bounds.max_size as usize; - - debug_assert!(a_bytes.len() <= a_max_size); - debug_assert!(b_bytes.len() <= b_max_size); - debug_assert!(c_bytes.len() <= c_max_size); - let a_size_len = bytes_to_store_size(&a_bounds) as usize; - let b_size_len = bytes_to_store_size(&b_bounds) as usize; - let c_size_len = bytes_to_store_size(&c_bounds) as usize; - let sizes_offset: usize = a_max_size + b_max_size + c_max_size; - - bytes[0..a_bytes.len()].copy_from_slice(a_bytes.borrow()); - - encode_size( - &mut bytes[sizes_offset..sizes_offset + a_size_len], - a_bytes.len(), - &a_bounds, - ); - bytes[a_max_size + a_size_len..a_max_size + a_size_len + b_bytes.len()] - .copy_from_slice(b_bytes.borrow()); - bytes[a_max_size + a_size_len + b_max_size - ..a_max_size + a_size_len + b_max_size + c_bytes.len()] - .copy_from_slice(c_bytes.borrow()); - - encode_size( - &mut bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], - b_bytes.len(), - &b_bounds, - ); - encode_size( - &mut bytes[sizes_offset + a_size_len + b_size_len - ..sizes_offset + a_size_len + b_size_len + c_size_len], - c_bytes.len(), - &c_bounds, - ); + let a_end = serialize_with_size(self.0.borrow(), &mut bytes, 0); + let b_end = serialize_with_size(self.1.borrow(), &mut bytes, a_end); + serialize_with_size(self.2.borrow(), &mut bytes, b_end); Cow::Owned(bytes) } _ => todo!("Serializing tuples with unbounded types is not yet supported."), @@ -586,16 +550,10 @@ where match Self::BOUND { Bound::Bounded { max_size, .. } => { assert_eq!(bytes.len(), max_size as usize); - let a_bounds = bounds::(); - let a_max_size = a_bounds.max_size as usize; - let a_size_len = bytes_to_store_size(&a_bounds) as usize; - let a_len = decode_size(&bytes[a_max_size..a_max_size + a_size_len], &a_bounds); - - let a = A::from_bytes(Cow::Borrowed(&bytes[0..a_len])); - let (b, c) = <(B, C)>::from_bytes(Cow::Borrowed( - &bytes[a_max_size + a_size_len..bytes.len()], - )); + let (a, a_end) = deserialize_with_size::(bytes.borrow(), 0); + let (b, b_end) = deserialize_with_size::(bytes.borrow(), a_end); + let (c, _) = deserialize_with_size::(bytes.borrow(), b_end); (a, b, c) } From 5dc1876e7643f6e7f883165243d71ba2ba6bc9db Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 15 Feb 2024 14:08:08 +0000 Subject: [PATCH 06/24] . --- src/storable.rs | 49 +++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index a068d416..8c9e435e 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -492,37 +492,40 @@ where }; } -fn serialize_with_size(entry: &T, bytes: &mut Vec, start: usize) -> usize +fn serialize_bounded_with_size(entry: &T, bytes: &mut [u8], start: usize) -> usize where T: Storable, { - let entry_bytes = entry.to_bytes(); - bytes[start..start + entry_bytes.len()].copy_from_slice(entry_bytes.borrow()); let bounds = bounds::(); + let size_len = bytes_to_store_size(&bounds) as usize; let max_size = bounds.max_size as usize; + let entry_bytes = entry.to_bytes(); debug_assert!(entry_bytes.len() <= max_size); - let size_len = bytes_to_store_size(&bounds) as usize; + encode_size( - &mut bytes[start + max_size..start + max_size + size_len], + &mut bytes[start..start + size_len], entry_bytes.len(), &bounds, ); + + bytes[start + size_len..start + size_len + entry_bytes.len()] + .copy_from_slice(entry_bytes.borrow()); + start + max_size + size_len } -fn deserialize_with_size(bytes: &Cow<[u8]>, start: usize) -> (T, usize) +fn deserialize_bounded_with_size(bytes: &[u8], start: usize) -> (T, usize) where T: Storable, { let bounds = bounds::(); let max_size = bounds.max_size as usize; let size_len = bytes_to_store_size(&bounds) as usize; - let len = decode_size( - &bytes[start + max_size..start + max_size + size_len], - &bounds, - ); + let len = decode_size(&bytes[start..start + size_len], &bounds); - let a = T::from_bytes(Cow::Borrowed(&bytes[start..start + len])); + let a = T::from_bytes(Cow::Borrowed( + &bytes[start + size_len..start + size_len + len], + )); (a, start + max_size + size_len) } @@ -537,9 +540,9 @@ where Bound::Bounded { max_size, .. } => { let mut bytes = vec![0; max_size as usize]; - let a_end = serialize_with_size(self.0.borrow(), &mut bytes, 0); - let b_end = serialize_with_size(self.1.borrow(), &mut bytes, a_end); - serialize_with_size(self.2.borrow(), &mut bytes, b_end); + let a_end = serialize_bounded_with_size(self.0.borrow(), &mut bytes, 0); + let b_end = serialize_bounded_with_size(self.1.borrow(), &mut bytes, a_end); + serialize_bounded_with_size(self.2.borrow(), &mut bytes, b_end); Cow::Owned(bytes) } _ => todo!("Serializing tuples with unbounded types is not yet supported."), @@ -551,9 +554,9 @@ where Bound::Bounded { max_size, .. } => { assert_eq!(bytes.len(), max_size as usize); - let (a, a_end) = deserialize_with_size::(bytes.borrow(), 0); - let (b, b_end) = deserialize_with_size::(bytes.borrow(), a_end); - let (c, _) = deserialize_with_size::(bytes.borrow(), b_end); + let (a, a_end) = deserialize_bounded_with_size::(bytes.borrow(), 0); + let (b, b_end) = deserialize_bounded_with_size::(bytes.borrow(), a_end); + let (c, _) = deserialize_bounded_with_size::(bytes.borrow(), b_end); (a, b, c) } @@ -565,13 +568,19 @@ where match (A::BOUND, B::BOUND, C::BOUND) { (Bound::Bounded { .. }, Bound::Bounded { .. }, Bound::Bounded { .. }) => { let a_bounds = bounds::(); - let b_c_bounds = bounds::<(B, C)>(); + let b_bounds = bounds::(); + let c_bounds = bounds::(); Bound::Bounded { max_size: a_bounds.max_size + bytes_to_store_size(&a_bounds) - + b_c_bounds.max_size, - is_fixed_size: a_bounds.is_fixed_size && b_c_bounds.is_fixed_size, + + b_bounds.max_size + + bytes_to_store_size(&b_bounds) + + c_bounds.max_size + + bytes_to_store_size(&c_bounds), + is_fixed_size: a_bounds.is_fixed_size + && b_bounds.is_fixed_size + && c_bounds.is_fixed_size, } } _ => Bound::Unbounded, From b68c9937f66e988b1b1f8b2fb5152956b785c33c Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 19 Feb 2024 14:56:32 +0000 Subject: [PATCH 07/24] add doc comment --- src/storable.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/storable.rs b/src/storable.rs index 8c9e435e..b9a67ce6 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -492,6 +492,11 @@ where }; } +/// Serializes `entry` into `bytes` starting from the index `start` +/// in `bytes`. +/// If serialized `entry` is saved in `bytes` on indices `[start, end)` +/// the function will return index `end` - the first index after +/// `start` that is not occupied with the serialization of `entry`. fn serialize_bounded_with_size(entry: &T, bytes: &mut [u8], start: usize) -> usize where T: Storable, @@ -514,6 +519,10 @@ where start + max_size + size_len } +/// Deserialize the struct starting at index `start` in `bytes`. +/// If serialized struct is saved in `bytes` on indices `[start, end)` the +/// function will return deserialized struct and index `end` - the first index +/// after `start` that is not occupied with the serialization of the struct. fn deserialize_bounded_with_size(bytes: &[u8], start: usize) -> (T, usize) where T: Storable, From 12a3eb4f55c3178c766b685f26e6651be035b85d Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 20 Feb 2024 12:43:06 +0000 Subject: [PATCH 08/24] do not encode size when the type has fixed size --- src/storable.rs | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index b9a67ce6..c2d15439 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -502,21 +502,27 @@ where T: Storable, { let bounds = bounds::(); - let size_len = bytes_to_store_size(&bounds) as usize; let max_size = bounds.max_size as usize; + let entry_bytes = entry.to_bytes(); debug_assert!(entry_bytes.len() <= max_size); - encode_size( - &mut bytes[start..start + size_len], - entry_bytes.len(), - &bounds, - ); + let length_of_size_encoding = if bounds.is_fixed_size { + 0 + } else { + let length_of_size_encoding = bytes_to_store_size(&bounds) as usize; + encode_size( + &mut bytes[start..start + length_of_size_encoding], + entry_bytes.len(), + &bounds, + ); + length_of_size_encoding + }; - bytes[start + size_len..start + size_len + entry_bytes.len()] + bytes[start + length_of_size_encoding..start + length_of_size_encoding + entry_bytes.len()] .copy_from_slice(entry_bytes.borrow()); - start + max_size + size_len + start + max_size + length_of_size_encoding } /// Deserialize the struct starting at index `start` in `bytes`. @@ -529,13 +535,19 @@ where { let bounds = bounds::(); let max_size = bounds.max_size as usize; - let size_len = bytes_to_store_size(&bounds) as usize; - let len = decode_size(&bytes[start..start + size_len], &bounds); + + let (length_of_size_encoding, actual_size) = if bounds.is_fixed_size { + (0, max_size) + } else { + let size_len = bytes_to_store_size(&bounds) as usize; + let len = decode_size(&bytes[start..start + size_len], &bounds); + (size_len, len) + }; let a = T::from_bytes(Cow::Borrowed( - &bytes[start + size_len..start + size_len + len], + &bytes[start + length_of_size_encoding..start + length_of_size_encoding + actual_size], )); - (a, start + max_size + size_len) + (a, start + max_size + length_of_size_encoding) } impl Storable for (A, B, C) From 9c1023459645e3cb3565cf3bc9dbc320b34963b7 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 21 Feb 2024 14:42:10 +0000 Subject: [PATCH 09/24] Revert "do not encode size when the type has fixed size" This reverts commit 12a3eb4f55c3178c766b685f26e6651be035b85d. --- src/storable.rs | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index c2d15439..b9a67ce6 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -502,27 +502,21 @@ where T: Storable, { let bounds = bounds::(); + let size_len = bytes_to_store_size(&bounds) as usize; let max_size = bounds.max_size as usize; - let entry_bytes = entry.to_bytes(); debug_assert!(entry_bytes.len() <= max_size); - let length_of_size_encoding = if bounds.is_fixed_size { - 0 - } else { - let length_of_size_encoding = bytes_to_store_size(&bounds) as usize; - encode_size( - &mut bytes[start..start + length_of_size_encoding], - entry_bytes.len(), - &bounds, - ); - length_of_size_encoding - }; + encode_size( + &mut bytes[start..start + size_len], + entry_bytes.len(), + &bounds, + ); - bytes[start + length_of_size_encoding..start + length_of_size_encoding + entry_bytes.len()] + bytes[start + size_len..start + size_len + entry_bytes.len()] .copy_from_slice(entry_bytes.borrow()); - start + max_size + length_of_size_encoding + start + max_size + size_len } /// Deserialize the struct starting at index `start` in `bytes`. @@ -535,19 +529,13 @@ where { let bounds = bounds::(); let max_size = bounds.max_size as usize; - - let (length_of_size_encoding, actual_size) = if bounds.is_fixed_size { - (0, max_size) - } else { - let size_len = bytes_to_store_size(&bounds) as usize; - let len = decode_size(&bytes[start..start + size_len], &bounds); - (size_len, len) - }; + let size_len = bytes_to_store_size(&bounds) as usize; + let len = decode_size(&bytes[start..start + size_len], &bounds); let a = T::from_bytes(Cow::Borrowed( - &bytes[start + length_of_size_encoding..start + length_of_size_encoding + actual_size], + &bytes[start + size_len..start + size_len + len], )); - (a, start + max_size + length_of_size_encoding) + (a, start + max_size + size_len) } impl Storable for (A, B, C) From 511be61345370897d7d32a529f7c0cc5b90687e6 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 21 Feb 2024 14:51:39 +0000 Subject: [PATCH 10/24] Fix comments --- src/storable.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index b9a67ce6..06b00ae2 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -492,11 +492,11 @@ where }; } -/// Serializes `entry` into `bytes` starting from the index `start` -/// in `bytes`. -/// If serialized `entry` is saved in `bytes` on indices `[start, end)` -/// the function will return index `end` - the first index after -/// `start` that is not occupied with the serialization of `entry`. +// Serializes `entry` into `bytes` starting from the index `start` +// in `bytes`. +// When serialized `entry` is saved in `bytes` on indices `[start, end)` +// the function will return index `end` - the first index after +// `start` that is not occupied with the serialization of `entry`. fn serialize_bounded_with_size(entry: &T, bytes: &mut [u8], start: usize) -> usize where T: Storable, @@ -520,7 +520,7 @@ where } /// Deserialize the struct starting at index `start` in `bytes`. -/// If serialized struct is saved in `bytes` on indices `[start, end)` the +/// When serialized struct is saved in `bytes` on indices `[start, end)` the /// function will return deserialized struct and index `end` - the first index /// after `start` that is not occupied with the serialization of the struct. fn deserialize_bounded_with_size(bytes: &[u8], start: usize) -> (T, usize) From 37b5cabf6c830d323be8850840e1f72f54c6358a Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 21 Feb 2024 14:52:08 +0000 Subject: [PATCH 11/24] Fix comments --- src/storable.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 06b00ae2..34bcaf5c 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -519,10 +519,10 @@ where start + max_size + size_len } -/// Deserialize the struct starting at index `start` in `bytes`. -/// When serialized struct is saved in `bytes` on indices `[start, end)` the -/// function will return deserialized struct and index `end` - the first index -/// after `start` that is not occupied with the serialization of the struct. +// Deserialize the struct starting at index `start` in `bytes`. +// When serialized struct is saved in `bytes` on indices `[start, end)` the +// function will return deserialized struct and index `end` - the first index +// after `start` that is not occupied with the serialization of the struct. fn deserialize_bounded_with_size(bytes: &[u8], start: usize) -> (T, usize) where T: Storable, From 822b9d095747b967e92433f62ca4d1c86e7c737b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 22 Feb 2024 10:25:36 +0000 Subject: [PATCH 12/24] Change encoding of (A, B, C) to take smallest possible footprint --- src/storable.rs | 60 +++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 34bcaf5c..a51c360f 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -497,26 +497,21 @@ where // When serialized `entry` is saved in `bytes` on indices `[start, end)` // the function will return index `end` - the first index after // `start` that is not occupied with the serialization of `entry`. -fn serialize_bounded_with_size(entry: &T, bytes: &mut [u8], start: usize) -> usize +fn serialize_bounded_with_size(entry_bytes: &[u8], bytes: &mut [u8], start: usize) -> usize where T: Storable, { let bounds = bounds::(); let size_len = bytes_to_store_size(&bounds) as usize; let max_size = bounds.max_size as usize; - let entry_bytes = entry.to_bytes(); debug_assert!(entry_bytes.len() <= max_size); + let actual_size = entry_bytes.len(); - encode_size( - &mut bytes[start..start + size_len], - entry_bytes.len(), - &bounds, - ); + encode_size(&mut bytes[start..start + size_len], actual_size, &bounds); - bytes[start + size_len..start + size_len + entry_bytes.len()] - .copy_from_slice(entry_bytes.borrow()); + bytes[start + size_len..start + size_len + actual_size].copy_from_slice(entry_bytes); - start + max_size + size_len + start + actual_size + size_len } // Deserialize the struct starting at index `start` in `bytes`. @@ -528,14 +523,26 @@ where T: Storable, { let bounds = bounds::(); - let max_size = bounds.max_size as usize; let size_len = bytes_to_store_size(&bounds) as usize; - let len = decode_size(&bytes[start..start + size_len], &bounds); + let actual_size = decode_size(&bytes[start..start + size_len], &bounds); let a = T::from_bytes(Cow::Borrowed( - &bytes[start + size_len..start + size_len + len], + &bytes[start + size_len..start + size_len + actual_size], )); - (a, start + max_size + size_len) + (a, start + actual_size + size_len) +} + +fn get_size_len() -> usize +where + T: Storable, +{ + match T::BOUND { + Bound::Bounded { .. } => { + let bounds = bounds::(); + bytes_to_store_size(&bounds) as usize + } + Bound::Unbounded => todo!("Serializing tuples with unbounded types is not yet supported."), + } } impl Storable for (A, B, C) @@ -546,12 +553,23 @@ where { fn to_bytes(&self) -> Cow<[u8]> { match Self::BOUND { - Bound::Bounded { max_size, .. } => { - let mut bytes = vec![0; max_size as usize]; + Bound::Bounded { .. } => { + let a_bytes = self.0.to_bytes(); + let b_bytes = self.1.to_bytes(); + let c_bytes = self.2.to_bytes(); + + let output_size = a_bytes.len() + + get_size_len::() + + b_bytes.len() + + get_size_len::() + + c_bytes.len() + + get_size_len::(); - let a_end = serialize_bounded_with_size(self.0.borrow(), &mut bytes, 0); - let b_end = serialize_bounded_with_size(self.1.borrow(), &mut bytes, a_end); - serialize_bounded_with_size(self.2.borrow(), &mut bytes, b_end); + let mut bytes = vec![0; output_size]; + + let a_end = serialize_bounded_with_size::(a_bytes.borrow(), &mut bytes, 0); + let b_end = serialize_bounded_with_size::(b_bytes.borrow(), &mut bytes, a_end); + serialize_bounded_with_size::(c_bytes.borrow(), &mut bytes, b_end); Cow::Owned(bytes) } _ => todo!("Serializing tuples with unbounded types is not yet supported."), @@ -560,9 +578,7 @@ where fn from_bytes(bytes: Cow<[u8]>) -> Self { match Self::BOUND { - Bound::Bounded { max_size, .. } => { - assert_eq!(bytes.len(), max_size as usize); - + Bound::Bounded { .. } => { let (a, a_end) = deserialize_bounded_with_size::(bytes.borrow(), 0); let (b, b_end) = deserialize_bounded_with_size::(bytes.borrow(), a_end); let (c, _) = deserialize_bounded_with_size::(bytes.borrow(), b_end); From 617ea8026d358c5a951ec3600ce954d95bad2830 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 22 Feb 2024 10:30:49 +0000 Subject: [PATCH 13/24] . --- src/storable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storable.rs b/src/storable.rs index a51c360f..a4ff3895 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -504,8 +504,8 @@ where let bounds = bounds::(); let size_len = bytes_to_store_size(&bounds) as usize; let max_size = bounds.max_size as usize; - debug_assert!(entry_bytes.len() <= max_size); let actual_size = entry_bytes.len(); + debug_assert!(actual_size <= max_size); encode_size(&mut bytes[start..start + size_len], actual_size, &bounds); From 21c2928677a37aed77db52cb2bb71e2236d2397a Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 22 Feb 2024 12:35:33 +0000 Subject: [PATCH 14/24] add encoding/decoding for unounded types --- src/storable.rs | 119 ++++++++++++++++++++++++------------------ src/storable/tests.rs | 22 ++++++++ 2 files changed, 91 insertions(+), 50 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index a4ff3895..41344fd1 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -492,22 +492,32 @@ where }; } +fn get_size_len() -> usize +where + T: Storable, +{ + match T::BOUND { + Bound::Bounded { .. } => { + let bounds = bounds::(); + bytes_to_store_size(&bounds) as usize + } + Bound::Unbounded => 8, + } +} + // Serializes `entry` into `bytes` starting from the index `start` // in `bytes`. // When serialized `entry` is saved in `bytes` on indices `[start, end)` // the function will return index `end` - the first index after // `start` that is not occupied with the serialization of `entry`. -fn serialize_bounded_with_size(entry_bytes: &[u8], bytes: &mut [u8], start: usize) -> usize +fn serialize_with_size(entry_bytes: &[u8], bytes: &mut [u8], start: usize) -> usize where T: Storable, { - let bounds = bounds::(); - let size_len = bytes_to_store_size(&bounds) as usize; - let max_size = bounds.max_size as usize; + let size_len = get_size_len::(); let actual_size = entry_bytes.len(); - debug_assert!(actual_size <= max_size); - encode_size(&mut bytes[start..start + size_len], actual_size, &bounds); + encode_size_universal::(&mut bytes[start..start + size_len], actual_size); bytes[start + size_len..start + size_len + actual_size].copy_from_slice(entry_bytes); @@ -522,9 +532,8 @@ fn deserialize_bounded_with_size(bytes: &[u8], start: usize) -> (T, usize) where T: Storable, { - let bounds = bounds::(); - let size_len = bytes_to_store_size(&bounds) as usize; - let actual_size = decode_size(&bytes[start..start + size_len], &bounds); + let size_len = get_size_len::(); + let actual_size = decode_size_universal::(&bytes[start..start + size_len]); let a = T::from_bytes(Cow::Borrowed( &bytes[start + size_len..start + size_len + actual_size], @@ -532,19 +541,6 @@ where (a, start + actual_size + size_len) } -fn get_size_len() -> usize -where - T: Storable, -{ - match T::BOUND { - Bound::Bounded { .. } => { - let bounds = bounds::(); - bytes_to_store_size(&bounds) as usize - } - Bound::Unbounded => todo!("Serializing tuples with unbounded types is not yet supported."), - } -} - impl Storable for (A, B, C) where A: Storable, @@ -552,41 +548,35 @@ where C: Storable, { fn to_bytes(&self) -> Cow<[u8]> { - match Self::BOUND { - Bound::Bounded { .. } => { - let a_bytes = self.0.to_bytes(); - let b_bytes = self.1.to_bytes(); - let c_bytes = self.2.to_bytes(); + let a_bytes = self.0.to_bytes(); + let b_bytes = self.1.to_bytes(); + let c_bytes = self.2.to_bytes(); - let output_size = a_bytes.len() - + get_size_len::() - + b_bytes.len() - + get_size_len::() - + c_bytes.len() - + get_size_len::(); + let output_size = a_bytes.len() + + get_size_len::() + + b_bytes.len() + + get_size_len::() + + c_bytes.len() + + get_size_len::(); - let mut bytes = vec![0; output_size]; + let mut bytes = vec![0; output_size]; - let a_end = serialize_bounded_with_size::(a_bytes.borrow(), &mut bytes, 0); - let b_end = serialize_bounded_with_size::(b_bytes.borrow(), &mut bytes, a_end); - serialize_bounded_with_size::(c_bytes.borrow(), &mut bytes, b_end); - Cow::Owned(bytes) - } - _ => todo!("Serializing tuples with unbounded types is not yet supported."), - } + let a_end = serialize_with_size::(a_bytes.borrow(), &mut bytes, 0); + let b_end = serialize_with_size::(b_bytes.borrow(), &mut bytes, a_end); + serialize_with_size::(c_bytes.borrow(), &mut bytes, b_end); + Cow::Owned(bytes) } fn from_bytes(bytes: Cow<[u8]>) -> Self { - match Self::BOUND { - Bound::Bounded { .. } => { - let (a, a_end) = deserialize_bounded_with_size::(bytes.borrow(), 0); - let (b, b_end) = deserialize_bounded_with_size::(bytes.borrow(), a_end); - let (c, _) = deserialize_bounded_with_size::(bytes.borrow(), b_end); - - (a, b, c) - } - _ => todo!("Deserializing tuples with unbounded types is not yet supported."), + if let Bound::Bounded { max_size, .. } = Self::BOUND { + assert!(bytes.len() <= max_size as usize); } + + let (a, a_end) = deserialize_bounded_with_size::(bytes.borrow(), 0); + let (b, b_end) = deserialize_bounded_with_size::(bytes.borrow(), a_end); + let (c, _) = deserialize_bounded_with_size::(bytes.borrow(), b_end); + + (a, b, c) } const BOUND: Bound = { @@ -715,6 +705,35 @@ fn encode_size(dst: &mut [u8], n: usize, bounds: &Bounds) { } } +fn decode_size_universal(src: &[u8]) -> usize +where + T: Storable, +{ + match T::BOUND { + Bound::Bounded { max_size, .. } => { + let size = decode_size(src, &bounds::()); + debug_assert!(size <= max_size as usize); + size + } + Bound::Unbounded => u64::from_be_bytes([ + src[0], src[1], src[2], src[3], src[4], src[5], src[6], src[7], + ]) as usize, + } +} + +fn encode_size_universal(dst: &mut [u8], n: usize) +where + T: Storable, +{ + match T::BOUND { + Bound::Bounded { max_size, .. } => { + debug_assert!(n <= max_size as usize); + encode_size(dst, n, &bounds::()) + } + Bound::Unbounded => dst[0..8].copy_from_slice(&(n as u64).to_be_bytes()), + } +} + pub(crate) const fn bytes_to_store_size(bounds: &Bounds) -> u32 { if bounds.is_fixed_size { 0 diff --git a/src/storable/tests.rs b/src/storable/tests.rs index 7746688c..4ff5ed92 100644 --- a/src/storable/tests.rs +++ b/src/storable/tests.rs @@ -21,6 +21,18 @@ proptest! { prop_assert_eq!(tuple, Storable::from_bytes(bytes)); } + #[test] + fn tuple_with_three_unbounded_elements_roundtrip(v1 in pvec(any::(), 0..4), v2 in pvec(any::(), 0..8), v3 in pvec(any::(), 0..12)) { + let tuple = (v1, v2, v3); + assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + + #[test] + fn tuple_with_three_elements_bounded_and_unbounded_roundtrip(v1 in pvec(any::(), 0..4), x in any::(), v2 in pvec(any::(), 0..12)) { + let tuple = (v1, x, v2); + assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + #[test] fn tuple_variable_width_u8_roundtrip(x in any::(), v in pvec(any::(), 0..40)) { @@ -83,6 +95,11 @@ proptest! { prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); } + #[test] + fn optional_tuple_with_three_unbounded_elements_roundtrip(v in proptest::option::of((pvec(any::(), 0..4), pvec(any::(), 0..8), pvec(any::(), 0..12)))) { + prop_assert_eq!(v.clone(), Storable::from_bytes(v.to_bytes())); + } + #[test] fn optional_tuple_variable_width_u8_roundtrip(v in proptest::option::of((any::(), pvec(any::(), 0..40)))) { let v = v.map(|(n, bytes)| (n, Blob::<48>::try_from(&bytes[..]).unwrap())); @@ -95,6 +112,11 @@ proptest! { prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); } + #[test] + fn optional_tuple_with_three_elements_bounded_and_unbounded_roundtrip(v in proptest::option::of((any::(), pvec(any::(), 0..40), pvec(any::(), 0..80)))) { + prop_assert_eq!(v.clone(), Storable::from_bytes(v.to_bytes())); + } + #[test] fn principal_roundtrip(mut bytes in pvec(any::(), 0..=28), tag in proptest::prop_oneof![Just(1),Just(2),Just(3),Just(4),Just(7)]) { bytes.push(tag); From 51d3c9cedf138ea1109b954a07a2901ad3e41e78 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 22 Feb 2024 12:48:05 +0000 Subject: [PATCH 15/24] . --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11613246..a5e78fc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: - name: Install DFX run: | wget --output-document install-dfx.sh "https://internetcomputer.org/install.sh" - DFX_VERSION=${DFX_VERSION:=0.14.3} bash install-dfx.sh < <(yes Y) + DFX_VERSION=${DFX_VERSION:=0.14.3} bash install-dfx.sh <<(yes Y) rm install-dfx.sh dfx cache install echo "$HOME/bin" >> $GITHUB_PATH From e5919e3c062a6dbf7a96b29e6a8f184807d4b7e0 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 22 Feb 2024 12:50:30 +0000 Subject: [PATCH 16/24] . --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5e78fc1..11613246 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: - name: Install DFX run: | wget --output-document install-dfx.sh "https://internetcomputer.org/install.sh" - DFX_VERSION=${DFX_VERSION:=0.14.3} bash install-dfx.sh <<(yes Y) + DFX_VERSION=${DFX_VERSION:=0.14.3} bash install-dfx.sh < <(yes Y) rm install-dfx.sh dfx cache install echo "$HOME/bin" >> $GITHUB_PATH From 0043f456644013aac5c0cdf3ea24281459d4462f Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 22 Feb 2024 14:24:47 +0000 Subject: [PATCH 17/24] refactor --- src/storable.rs | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 41344fd1..11930554 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -497,10 +497,13 @@ where T: Storable, { match T::BOUND { - Bound::Bounded { .. } => { - let bounds = bounds::(); - bytes_to_store_size(&bounds) as usize - } + Bound::Bounded { + max_size, + is_fixed_size, + } => bytes_to_store_size(&Bounds { + max_size, + is_fixed_size, + }) as usize, Bound::Unbounded => 8, } } @@ -710,8 +713,17 @@ where T: Storable, { match T::BOUND { - Bound::Bounded { max_size, .. } => { - let size = decode_size(src, &bounds::()); + Bound::Bounded { + max_size, + is_fixed_size, + } => { + let size = decode_size( + src, + &Bounds { + max_size, + is_fixed_size, + }, + ); debug_assert!(size <= max_size as usize); size } @@ -726,9 +738,19 @@ where T: Storable, { match T::BOUND { - Bound::Bounded { max_size, .. } => { + Bound::Bounded { + max_size, + is_fixed_size, + } => { debug_assert!(n <= max_size as usize); - encode_size(dst, n, &bounds::()) + encode_size( + dst, + n, + &Bounds { + max_size, + is_fixed_size, + }, + ) } Bound::Unbounded => dst[0..8].copy_from_slice(&(n as u64).to_be_bytes()), } From ee3ec621baf15674ddc0e66a19d7fc5c9ea061c3 Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Mon, 26 Feb 2024 12:50:50 +0100 Subject: [PATCH 18/24] Update src/storable.rs Co-authored-by: Islam El-Ashi --- src/storable.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storable.rs b/src/storable.rs index 11930554..33dba28e 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -551,6 +551,7 @@ where C: Storable, { fn to_bytes(&self) -> Cow<[u8]> { + // Serialize each of the tuple's elements let a_bytes = self.0.to_bytes(); let b_bytes = self.1.to_bytes(); let c_bytes = self.2.to_bytes(); From d58b37b0ee623bcac1d0b02beb91d13ad2457610 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 26 Feb 2024 11:57:33 +0000 Subject: [PATCH 19/24] rename --- src/storable.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 33dba28e..0eb6d151 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -492,7 +492,7 @@ where }; } -fn get_size_len() -> usize +fn get_num_bytes_required_to_store_size() -> usize where T: Storable, { @@ -517,7 +517,7 @@ fn serialize_with_size(entry_bytes: &[u8], bytes: &mut [u8], start: usize) -> where T: Storable, { - let size_len = get_size_len::(); + let size_len = get_num_bytes_required_to_store_size::(); let actual_size = entry_bytes.len(); encode_size_universal::(&mut bytes[start..start + size_len], actual_size); @@ -535,7 +535,7 @@ fn deserialize_bounded_with_size(bytes: &[u8], start: usize) -> (T, usize) where T: Storable, { - let size_len = get_size_len::(); + let size_len = get_num_bytes_required_to_store_size::(); let actual_size = decode_size_universal::(&bytes[start..start + size_len]); let a = T::from_bytes(Cow::Borrowed( @@ -557,11 +557,11 @@ where let c_bytes = self.2.to_bytes(); let output_size = a_bytes.len() - + get_size_len::() + + get_num_bytes_required_to_store_size::() + b_bytes.len() - + get_size_len::() + + get_num_bytes_required_to_store_size::() + c_bytes.len() - + get_size_len::(); + + get_num_bytes_required_to_store_size::(); let mut bytes = vec![0; output_size]; From 3eeaf15766ceb0659f3eb99e070fc3fdd76a8b98 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 26 Feb 2024 12:01:10 +0000 Subject: [PATCH 20/24] . --- src/storable.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 0eb6d151..93f40b87 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -531,7 +531,7 @@ where // When serialized struct is saved in `bytes` on indices `[start, end)` the // function will return deserialized struct and index `end` - the first index // after `start` that is not occupied with the serialization of the struct. -fn deserialize_bounded_with_size(bytes: &[u8], start: usize) -> (T, usize) +fn deserialize_with_size(bytes: &[u8], start: usize) -> (T, usize) where T: Storable, { @@ -576,9 +576,9 @@ where assert!(bytes.len() <= max_size as usize); } - let (a, a_end) = deserialize_bounded_with_size::(bytes.borrow(), 0); - let (b, b_end) = deserialize_bounded_with_size::(bytes.borrow(), a_end); - let (c, _) = deserialize_bounded_with_size::(bytes.borrow(), b_end); + let (a, a_end) = deserialize_with_size::(bytes.borrow(), 0); + let (b, b_end) = deserialize_with_size::(bytes.borrow(), a_end); + let (c, _) = deserialize_with_size::(bytes.borrow(), b_end); (a, b, c) } From e946631f50ae13d5326ca7083443f90c2e287fd6 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 26 Feb 2024 12:02:57 +0000 Subject: [PATCH 21/24] rename --- src/storable.rs | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 93f40b87..d7b282c1 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -424,12 +424,12 @@ where let sizes_offset: usize = a_max_size + b_max_size; - encode_size( + encode_size_of_bound( &mut bytes[sizes_offset..sizes_offset + a_size_len], a_bytes.len(), &a_bounds, ); - encode_size( + encode_size_of_bound( &mut bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], b_bytes.len(), &b_bounds, @@ -454,8 +454,11 @@ where let a_size_len = bytes_to_store_size(&a_bounds) as usize; let b_size_len = bytes_to_store_size(&b_bounds) as usize; - let a_len = decode_size(&bytes[sizes_offset..sizes_offset + a_size_len], &a_bounds); - let b_len = decode_size( + let a_len = decode_size_of_bound( + &bytes[sizes_offset..sizes_offset + a_size_len], + &a_bounds, + ); + let b_len = decode_size_of_bound( &bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], &b_bounds, ); @@ -520,7 +523,7 @@ where let size_len = get_num_bytes_required_to_store_size::(); let actual_size = entry_bytes.len(); - encode_size_universal::(&mut bytes[start..start + size_len], actual_size); + encode_size::(&mut bytes[start..start + size_len], actual_size); bytes[start + size_len..start + size_len + actual_size].copy_from_slice(entry_bytes); @@ -536,7 +539,7 @@ where T: Storable, { let size_len = get_num_bytes_required_to_store_size::(); - let actual_size = decode_size_universal::(&bytes[start..start + size_len]); + let actual_size = decode_size::(&bytes[start..start + size_len]); let a = T::from_bytes(Cow::Borrowed( &bytes[start + size_len..start + size_len + actual_size], @@ -683,7 +686,7 @@ pub(crate) const fn bounds() -> Bounds { } } -fn decode_size(src: &[u8], bounds: &Bounds) -> usize { +fn decode_size_of_bound(src: &[u8], bounds: &Bounds) -> usize { if bounds.is_fixed_size { bounds.max_size as usize } else if bounds.max_size <= u8::MAX as u32 { @@ -695,7 +698,7 @@ fn decode_size(src: &[u8], bounds: &Bounds) -> usize { } } -fn encode_size(dst: &mut [u8], n: usize, bounds: &Bounds) { +fn encode_size_of_bound(dst: &mut [u8], n: usize, bounds: &Bounds) { if bounds.is_fixed_size { return; } @@ -709,7 +712,7 @@ fn encode_size(dst: &mut [u8], n: usize, bounds: &Bounds) { } } -fn decode_size_universal(src: &[u8]) -> usize +fn decode_size(src: &[u8]) -> usize where T: Storable, { @@ -718,7 +721,7 @@ where max_size, is_fixed_size, } => { - let size = decode_size( + let size = decode_size_of_bound( src, &Bounds { max_size, @@ -734,7 +737,7 @@ where } } -fn encode_size_universal(dst: &mut [u8], n: usize) +fn encode_size(dst: &mut [u8], n: usize) where T: Storable, { @@ -744,7 +747,7 @@ where is_fixed_size, } => { debug_assert!(n <= max_size as usize); - encode_size( + encode_size_of_bound( dst, n, &Bounds { From 177a12d402a4232d9a9051e0a0907b2dcdc6d364 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 26 Feb 2024 12:22:03 +0000 Subject: [PATCH 22/24] refactor --- src/storable.rs | 61 ++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index d7b282c1..a3d2810d 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -495,22 +495,6 @@ where }; } -fn get_num_bytes_required_to_store_size() -> usize -where - T: Storable, -{ - match T::BOUND { - Bound::Bounded { - max_size, - is_fixed_size, - } => bytes_to_store_size(&Bounds { - max_size, - is_fixed_size, - }) as usize, - Bound::Unbounded => 8, - } -} - // Serializes `entry` into `bytes` starting from the index `start` // in `bytes`. // When serialized `entry` is saved in `bytes` on indices `[start, end)` @@ -686,6 +670,36 @@ pub(crate) const fn bounds() -> Bounds { } } +pub(crate) const fn bytes_to_store_size(bounds: &Bounds) -> u32 { + if bounds.is_fixed_size { + 0 + } else if bounds.max_size <= u8::MAX as u32 { + 1 + } else if bounds.max_size <= u16::MAX as u32 { + 2 + } else { + 4 + } +} + +const NUM_BYTES_TO_STORE_SIZE_OF_UNBOUNDED_TYPE: usize = 8; + +const fn get_num_bytes_required_to_store_size() -> usize +where + T: Storable, +{ + match T::BOUND { + Bound::Bounded { + max_size, + is_fixed_size, + } => bytes_to_store_size(&Bounds { + max_size, + is_fixed_size, + }) as usize, + Bound::Unbounded => NUM_BYTES_TO_STORE_SIZE_OF_UNBOUNDED_TYPE, + } +} + fn decode_size_of_bound(src: &[u8], bounds: &Bounds) -> usize { if bounds.is_fixed_size { bounds.max_size as usize @@ -756,18 +770,7 @@ where }, ) } - Bound::Unbounded => dst[0..8].copy_from_slice(&(n as u64).to_be_bytes()), - } -} - -pub(crate) const fn bytes_to_store_size(bounds: &Bounds) -> u32 { - if bounds.is_fixed_size { - 0 - } else if bounds.max_size <= u8::MAX as u32 { - 1 - } else if bounds.max_size <= u16::MAX as u32 { - 2 - } else { - 4 + Bound::Unbounded => dst[0..NUM_BYTES_TO_STORE_SIZE_OF_UNBOUNDED_TYPE] + .copy_from_slice(&(n as u64).to_be_bytes()), } } From f9cddc2a01d2f6ff58a02c1e6eeaefde5d0af518 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 26 Feb 2024 12:29:31 +0000 Subject: [PATCH 23/24] rename --- src/storable.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index a3d2810d..7a469451 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -495,12 +495,12 @@ where }; } -// Serializes `entry` into `bytes` starting from the index `start` -// in `bytes`. -// When serialized `entry` is saved in `bytes` on indices `[start, end)` +// Encodes `entry_bytes` size followed with `entry_bytes` into `bytes` +// starting from the index `start`. +// When encoding is saved in `bytes` on indices `[start, end)` // the function will return index `end` - the first index after -// `start` that is not occupied with the serialization of `entry`. -fn serialize_with_size(entry_bytes: &[u8], bytes: &mut [u8], start: usize) -> usize +// `start` that is not occupied with encoding. +fn encode_with_size(entry_bytes: &[u8], bytes: &mut [u8], start: usize) -> usize where T: Storable, { @@ -552,9 +552,9 @@ where let mut bytes = vec![0; output_size]; - let a_end = serialize_with_size::(a_bytes.borrow(), &mut bytes, 0); - let b_end = serialize_with_size::(b_bytes.borrow(), &mut bytes, a_end); - serialize_with_size::(c_bytes.borrow(), &mut bytes, b_end); + let a_end = encode_with_size::(a_bytes.borrow(), &mut bytes, 0); + let b_end = encode_with_size::(b_bytes.borrow(), &mut bytes, a_end); + encode_with_size::(c_bytes.borrow(), &mut bytes, b_end); Cow::Owned(bytes) } From 6432778bba27cb4058abf37ab6569b755d11740b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 27 Feb 2024 15:06:03 +0000 Subject: [PATCH 24/24] size of lenght of Unbounded types to 4 B --- src/storable.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 7a469451..ebc177bd 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -682,7 +682,7 @@ pub(crate) const fn bytes_to_store_size(bounds: &Bounds) -> u32 { } } -const NUM_BYTES_TO_STORE_SIZE_OF_UNBOUNDED_TYPE: usize = 8; +const NUM_BYTES_TO_STORE_SIZE_OF_UNBOUNDED_TYPE: usize = 4; const fn get_num_bytes_required_to_store_size() -> usize where @@ -745,9 +745,7 @@ where debug_assert!(size <= max_size as usize); size } - Bound::Unbounded => u64::from_be_bytes([ - src[0], src[1], src[2], src[3], src[4], src[5], src[6], src[7], - ]) as usize, + Bound::Unbounded => u32::from_be_bytes([src[0], src[1], src[2], src[3]]) as usize, } } @@ -771,6 +769,6 @@ where ) } Bound::Unbounded => dst[0..NUM_BYTES_TO_STORE_SIZE_OF_UNBOUNDED_TYPE] - .copy_from_slice(&(n as u64).to_be_bytes()), + .copy_from_slice(&(n as u32).to_be_bytes()), } }