From ad838b3e2af0279c6372fa1a479066474e8ad235 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 5 Mar 2024 16:13:58 +0000 Subject: [PATCH 01/17] . --- src/storable.rs | 154 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 6 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index dfcb55e8..a2437c9f 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -424,12 +424,12 @@ where let sizes_offset: usize = a_max_size + b_max_size; - encode_size( + encode_size_of_bound( &mut bytes[sizes_offset..sizes_offset + a_size_len], a_bytes.len(), &a_bounds, ); - encode_size( + encode_size_of_bound( &mut bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], b_bytes.len(), &b_bounds, @@ -454,8 +454,11 @@ where let a_size_len = bytes_to_store_size(&a_bounds) as usize; let b_size_len = bytes_to_store_size(&b_bounds) as usize; - let a_len = decode_size(&bytes[sizes_offset..sizes_offset + a_size_len], &a_bounds); - let b_len = decode_size( + let a_len = decode_size_of_bound( + &bytes[sizes_offset..sizes_offset + a_size_len], + &a_bounds, + ); + let b_len = decode_size_of_bound( &bytes[sizes_offset + a_size_len..sizes_offset + a_size_len + b_size_len], &b_bounds, ); @@ -568,7 +571,7 @@ pub(crate) const fn bounds() -> Bounds { } } -fn decode_size(src: &[u8], bounds: &Bounds) -> usize { +fn decode_size_of_bound(src: &[u8], bounds: &Bounds) -> usize { if bounds.is_fixed_size { bounds.max_size as usize } else if bounds.max_size <= u8::MAX as u32 { @@ -580,7 +583,7 @@ fn decode_size(src: &[u8], bounds: &Bounds) -> usize { } } -fn encode_size(dst: &mut [u8], n: usize, bounds: &Bounds) { +fn encode_size_of_bound(dst: &mut [u8], n: usize, bounds: &Bounds) { if bounds.is_fixed_size { return; } @@ -594,6 +597,22 @@ fn encode_size(dst: &mut [u8], n: usize, bounds: &Bounds) { } } +fn decode_size(src: &[u8], size_len: usize) -> usize { + match size_len { + 1 => src[0] as usize, + 2 => u16::from_be_bytes([src[0], src[1]]) as usize, + _ => u32::from_be_bytes([src[0], src[1], src[2], src[3]]) as usize, + } +} + +fn encode_size(dst: &mut [u8], n: usize, size_len: usize) { + match size_len { + 1 => dst[0] = n as u8, + 2 => dst[0..2].copy_from_slice(&(n as u16).to_be_bytes()), + _ => dst[0..4].copy_from_slice(&(n as u32).to_be_bytes()), + }; +} + pub(crate) const fn bytes_to_store_size(bounds: &Bounds) -> u32 { if bounds.is_fixed_size { 0 @@ -605,3 +624,126 @@ pub(crate) const fn bytes_to_store_size(bounds: &Bounds) -> u32 { 4 } } + +fn size_len(bytes_size: usize) -> usize { + if bytes_size <= u8::MAX as usize { + 1 + } else if bytes_size <= u16::MAX as usize { + 2 + } else { + 4 + } +} + +fn get_size_len_byte(sizes_len: Vec) -> u8 { + if sizes_len.len() > 4 { + panic!(""); + } + let mut sizes_len_byte: u8 = 0; + + for size_len in sizes_len.iter() { + if *size_len > 4 { + panic!(); + } + sizes_len_byte <<= 2; + sizes_len_byte += (*size_len - 1) as u8; + } + + sizes_len_byte +} + +fn get_size_lens(mut sizes_len_byte: u8, number_of_elements: u8) -> Vec { + if number_of_elements > 5 { + panic!(""); + } + let mut size_lens = vec![]; + for i in 0..number_of_elements - 1 { + let mask: u8 = 3; + let curr_size: u8 = (sizes_len_byte & mask) + 1; + size_lens.push(curr_size); + sizes_len_byte >>= 2; + } + + size_lens.reverse(); + + size_lens +} + +impl Storable for (A, B, C) +where + A: Storable, + B: Storable, + C: Storable, +{ + fn to_bytes(&self) -> Cow<[u8]> { + let a_bytes = self.0.to_bytes(); + let a_size = a_bytes.len(); + let a_size_len = size_len(a_size); + let b_bytes = self.1.to_bytes(); + let b_size = b_bytes.len(); + let b_size_len = size_len(b_size); + let c_bytes = self.1.to_bytes(); + let c_size = c_bytes.len(); + + let output_size = a_size + b_size + c_size + a_size_len + b_size_len + 1; + + let mut bytes = vec![0; output_size]; + let mut curr_ind = 0; + bytes[0] = get_size_len_byte(vec![a_size_len, b_size_len]); + curr_ind += 1; + encode_size(&mut bytes[curr_ind..], a_size, a_size_len); + curr_ind += a_size_len; + bytes[curr_ind..curr_ind + a_size].copy_from_slice(a_bytes.borrow()); + curr_ind += a_size; + encode_size(&mut bytes[curr_ind..], b_size, b_size_len); + curr_ind += b_size_len; + bytes[curr_ind..curr_ind + b_size].copy_from_slice(b_bytes.borrow()); + curr_ind += b_size; + bytes[curr_ind..curr_ind + c_size].copy_from_slice(c_bytes.borrow()); + debug_assert_eq!(curr_ind + c_size, output_size); + + Cow::Owned(bytes) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + let mut curr_ind = 0; + let sizes_len = get_size_lens(bytes[0], 3); + curr_ind += 1; + let a_size_len = sizes_len[0] as usize; + let a_size = decode_size(&bytes[curr_ind..], a_size_len); + curr_ind += a_size_len; + let a = A::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + a_size])); + curr_ind += a_size; + let b_size_len = sizes_len[1] as usize; + let b_size = decode_size(&bytes[curr_ind..], b_size_len); + curr_ind += b_size_len; + let b = B::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + b_size])); + curr_ind += b_size; + let c = C::from_bytes(Cow::Borrowed(&bytes[curr_ind..])); + + (a, b, c) + } + + const BOUND: Bound = { + match (A::BOUND, B::BOUND, C::BOUND) { + (Bound::Bounded { .. }, Bound::Bounded { .. }, Bound::Bounded { .. }) => { + let a_bounds = bounds::(); + let b_bounds = bounds::(); + let c_bounds = bounds::(); + + Bound::Bounded { + max_size: a_bounds.max_size + + bytes_to_store_size(&a_bounds) + + b_bounds.max_size + + bytes_to_store_size(&b_bounds) + + c_bounds.max_size + + bytes_to_store_size(&c_bounds), + is_fixed_size: a_bounds.is_fixed_size + && b_bounds.is_fixed_size + && c_bounds.is_fixed_size, + } + } + _ => Bound::Unbounded, + } + }; +} From eda9ca46a501345ffcdfaee272baf1413fe3c5cf Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 5 Mar 2024 16:24:44 +0000 Subject: [PATCH 02/17] . --- proptest-regressions/storable/tests.txt | 7 +++ src/storable.rs | 4 +- src/storable/tests.rs | 60 +++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 proptest-regressions/storable/tests.txt diff --git a/proptest-regressions/storable/tests.txt b/proptest-regressions/storable/tests.txt new file mode 100644 index 00000000..7b0a415e --- /dev/null +++ b/proptest-regressions/storable/tests.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc a93f6556dd7870c8385191a803f08c9bb2afb28ed66fe2a5266c2f6faa3cf438 # shrinks to v = Some((0, [], [0])) diff --git a/src/storable.rs b/src/storable.rs index a2437c9f..a3621e4a 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -657,7 +657,7 @@ fn get_size_lens(mut sizes_len_byte: u8, number_of_elements: u8) -> Vec { panic!(""); } let mut size_lens = vec![]; - for i in 0..number_of_elements - 1 { + for _ in 0..number_of_elements - 1 { let mask: u8 = 3; let curr_size: u8 = (sizes_len_byte & mask) + 1; size_lens.push(curr_size); @@ -682,7 +682,7 @@ where let b_bytes = self.1.to_bytes(); let b_size = b_bytes.len(); let b_size_len = size_len(b_size); - let c_bytes = self.1.to_bytes(); + let c_bytes = self.2.to_bytes(); let c_size = c_bytes.len(); let output_size = a_size + b_size + c_size + a_size_len + b_size_len + 1; diff --git a/src/storable/tests.rs b/src/storable/tests.rs index e04c55e5..2e12426e 100644 --- a/src/storable/tests.rs +++ b/src/storable/tests.rs @@ -13,6 +13,28 @@ proptest! { prop_assert_eq!(tuple, Storable::from_bytes(bytes)); } + #[test] + fn tuple_with_three_elements_roundtrip(x in any::(), y in uniform20(any::()), z in uniform20(any::())) { + let tuple = (x, y, z); + let bytes = tuple.to_bytes(); + // 1B sizes len | 1B x size | 8B x bytes | 1B y size | 20B y bytes | 20B z bytes + prop_assert_eq!(bytes.len(), 51); + prop_assert_eq!(tuple, Storable::from_bytes(bytes)); + } + + #[test] + fn tuple_with_three_unbounded_elements_roundtrip(v1 in pvec(any::(), 0..4), v2 in pvec(any::(), 0..8), v3 in pvec(any::(), 0..12)) { + let tuple = (v1, v2, v3); + assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + + #[test] + fn tuple_with_three_elements_bounded_and_unbounded_roundtrip(v1 in pvec(any::(), 0..4), x in any::(), v2 in pvec(any::(), 0..12)) { + let tuple = (v1, x, v2); + assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + + #[test] fn tuple_variable_width_u8_roundtrip(x in any::(), v in pvec(any::(), 0..40)) { let bytes = Blob::<48>::try_from(&v[..]).unwrap(); @@ -20,6 +42,14 @@ proptest! { prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); } + #[test] + fn tuple_with_three_elements_variable_width_u8_roundtrip(x in any::(), v1 in pvec(any::(), 0..40), v2 in pvec(any::(), 0..80)) { + let v1_bytes = Blob::<40>::try_from(&v1[..]).unwrap(); + let v2_bytes = Blob::<80>::try_from(&v2[..]).unwrap(); + let tuple = (x, v1_bytes, v2_bytes); + prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + #[test] fn tuple_variable_width_u16_roundtrip(x in any::(), v in pvec(any::(), 0..40)) { let bytes = Blob::<300>::try_from(&v[..]).unwrap(); @@ -27,6 +57,15 @@ proptest! { prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); } + #[test] + fn tuple_with_three_elements_variable_width_u16_roundtrip(x in any::(), v1 in pvec(any::(), 0..40), v2 in pvec(any::(), 0..80)) { + let v1_bytes = Blob::<300>::try_from(&v1[..]).unwrap(); + let v2_bytes = Blob::<300>::try_from(&v2[..]).unwrap(); + + let tuple = (x, v1_bytes, v2_bytes); + prop_assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + } + #[test] fn f64_roundtrip(v in any::()) { prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); @@ -52,12 +91,33 @@ proptest! { prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); } + #[test] + fn optional_tuple_with_three_elements_roundtrip(v in proptest::option::of((any::(), uniform20(any::()), uniform20(any::())))) { + prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); + } + + #[test] + fn optional_tuple_with_three_unbounded_elements_roundtrip(v in proptest::option::of((pvec(any::(), 0..4), pvec(any::(), 0..8), pvec(any::(), 0..12)))) { + prop_assert_eq!(v.clone(), Storable::from_bytes(v.to_bytes())); + } + #[test] fn optional_tuple_variable_width_u8_roundtrip(v in proptest::option::of((any::(), pvec(any::(), 0..40)))) { let v = v.map(|(n, bytes)| (n, Blob::<48>::try_from(&bytes[..]).unwrap())); prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); } + #[test] + fn optional_tuple_with_three_elements_variable_width_u8_roundtrip(v in proptest::option::of((any::(), pvec(any::(), 0..40), pvec(any::(), 0..80)))) { + let v = v.map(|(n, bytes_1, bytes_2)| (n, Blob::<40>::try_from(&bytes_1[..]).unwrap(), Blob::<80>::try_from(&bytes_2[..]).unwrap())); + prop_assert_eq!(v, Storable::from_bytes(v.to_bytes())); + } + + #[test] + fn optional_tuple_with_three_elements_bounded_and_unbounded_roundtrip(v in proptest::option::of((any::(), pvec(any::(), 0..40), pvec(any::(), 0..80)))) { + prop_assert_eq!(v.clone(), Storable::from_bytes(v.to_bytes())); + } + #[test] fn principal_roundtrip(mut bytes in pvec(any::(), 0..=28), tag in proptest::prop_oneof![Just(1),Just(2),Just(3),Just(4),Just(7)]) { bytes.push(tag); From 2797d046ab91ca1c4acee35f027f1e1906539845 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 5 Mar 2024 16:26:29 +0000 Subject: [PATCH 03/17] . --- proptest-regressions/storable/tests.txt | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 proptest-regressions/storable/tests.txt diff --git a/proptest-regressions/storable/tests.txt b/proptest-regressions/storable/tests.txt deleted file mode 100644 index 7b0a415e..00000000 --- a/proptest-regressions/storable/tests.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc a93f6556dd7870c8385191a803f08c9bb2afb28ed66fe2a5266c2f6faa3cf438 # shrinks to v = Some((0, [], [0])) From 3d094e3fccc39b8baf6a42b80ed8e207764f7ade Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 6 Mar 2024 12:47:00 +0000 Subject: [PATCH 04/17] add comments --- src/base_vec.rs | 4 +- src/storable.rs | 118 ++++++++++++++++++++++++++++-------------------- 2 files changed, 70 insertions(+), 52 deletions(-) diff --git a/src/base_vec.rs b/src/base_vec.rs index c99d4a70..af6232ae 100644 --- a/src/base_vec.rs +++ b/src/base_vec.rs @@ -31,7 +31,7 @@ //! type is fixed in size, the `SLOT_SIZE` is equal to the max size. //! Otherwise, the `SLOT_SIZE` is the max size plus the number of //! bytes required to represent integers up to that max size. -use crate::storable::{bounds, bytes_to_store_size}; +use crate::storable::{bounds, bytes_to_store_size_bounded}; use crate::{ read_u32, read_u64, safe_write, write_u32, write_u64, Address, GrowFailed, Memory, Storable, }; @@ -341,7 +341,7 @@ impl fmt::Debug for BaseVec { fn slot_size() -> u32 { let t_bounds = bounds::(); - t_bounds.max_size + bytes_to_store_size(&t_bounds) + t_bounds.max_size + bytes_to_store_size_bounded(&t_bounds) } pub struct Iter<'a, T, M> diff --git a/src/storable.rs b/src/storable.rs index a3621e4a..61db2377 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -419,8 +419,8 @@ where bytes[0..a_bytes.len()].copy_from_slice(a_bytes.borrow()); bytes[a_max_size..a_max_size + b_bytes.len()].copy_from_slice(b_bytes.borrow()); - let a_size_len = bytes_to_store_size(&a_bounds) as usize; - let b_size_len = bytes_to_store_size(&b_bounds) as usize; + let a_size_len = bytes_to_store_size_bounded(&a_bounds) as usize; + let b_size_len = bytes_to_store_size_bounded(&b_bounds) as usize; let sizes_offset: usize = a_max_size + b_max_size; @@ -452,8 +452,8 @@ where let b_max_size = b_bounds.max_size as usize; let sizes_offset = a_max_size + b_max_size; - let a_size_len = bytes_to_store_size(&a_bounds) as usize; - let b_size_len = bytes_to_store_size(&b_bounds) as usize; + let a_size_len = bytes_to_store_size_bounded(&a_bounds) as usize; + let b_size_len = bytes_to_store_size_bounded(&b_bounds) as usize; let a_len = decode_size_of_bound( &bytes[sizes_offset..sizes_offset + a_size_len], &a_bounds, @@ -480,8 +480,8 @@ where let max_size = a_bounds.max_size + b_bounds.max_size - + bytes_to_store_size(&a_bounds) - + bytes_to_store_size(&b_bounds); + + bytes_to_store_size_bounded(&a_bounds) + + bytes_to_store_size_bounded(&b_bounds); let is_fixed_size = a_bounds.is_fixed_size && b_bounds.is_fixed_size; @@ -605,27 +605,25 @@ fn decode_size(src: &[u8], size_len: usize) -> usize { } } -fn encode_size(dst: &mut [u8], n: usize, size_len: usize) { - match size_len { +fn encode_size(dst: &mut [u8], n: usize) -> usize { + let bytes_to_store_size = bytes_to_store_size(n); + match bytes_to_store_size { 1 => dst[0] = n as u8, 2 => dst[0..2].copy_from_slice(&(n as u16).to_be_bytes()), _ => dst[0..4].copy_from_slice(&(n as u32).to_be_bytes()), }; + bytes_to_store_size } -pub(crate) const fn bytes_to_store_size(bounds: &Bounds) -> u32 { +pub(crate) const fn bytes_to_store_size_bounded(bounds: &Bounds) -> u32 { if bounds.is_fixed_size { 0 - } else if bounds.max_size <= u8::MAX as u32 { - 1 - } else if bounds.max_size <= u16::MAX as u32 { - 2 } else { - 4 + bytes_to_store_size(bounds.max_size as usize) as u32 } } -fn size_len(bytes_size: usize) -> usize { +const fn bytes_to_store_size(bytes_size: usize) -> usize { if bytes_size <= u8::MAX as usize { 1 } else if bytes_size <= u16::MAX as usize { @@ -635,38 +633,49 @@ fn size_len(bytes_size: usize) -> usize { } } -fn get_size_len_byte(sizes_len: Vec) -> u8 { - if sizes_len.len() > 4 { - panic!(""); - } - let mut sizes_len_byte: u8 = 0; +fn encode_size_lengths(sizes: Vec) -> u8 { + assert!(sizes.len() <= 4); - for size_len in sizes_len.iter() { - if *size_len > 4 { - panic!(); - } - sizes_len_byte <<= 2; - sizes_len_byte += (*size_len - 1) as u8; + let mut size_lengths_byte: u8 = 0; + + for size in sizes.iter() { + let size_length = bytes_to_store_size(*size); + // Number of bytes required to store the size of every + // element is represented with 2 bits. + size_lengths_byte <<= 2; + // `size_length` can take value in {1, 2, 4}, but to + // compress it into 2 bit we will decrement its value. + size_lengths_byte += (size_length - 1) as u8; } - sizes_len_byte + size_lengths_byte } -fn get_size_lens(mut sizes_len_byte: u8, number_of_elements: u8) -> Vec { - if number_of_elements > 5 { - panic!(""); - } - let mut size_lens = vec![]; - for _ in 0..number_of_elements - 1 { +fn decode_sizes_length(mut encoded_bytes_to_store: u8, number_of_encoded_lengths: u8) -> Vec { + assert!(number_of_encoded_lengths <= 4); + + let mut bytes_to_store_sizes = vec![]; + + for _ in 0..number_of_encoded_lengths - 1 { + // The number of bytes required to store the size of every + // element is represented with 2 bits. Hence we use + // mask `11`, equivalent to 3 in the decimal system. let mask: u8 = 3; - let curr_size: u8 = (sizes_len_byte & mask) + 1; - size_lens.push(curr_size); - sizes_len_byte >>= 2; + // The number of bytes required to store size can take value + // in {1, 2, 4}, but to compress it to 2-bit, + // when encoding we decreased the value, hence now we need + // to do inverse. + let bytes_to_store: u8 = (encoded_bytes_to_store & mask) + 1; + bytes_to_store_sizes.push(bytes_to_store); + encoded_bytes_to_store >>= 2; } - size_lens.reverse(); + // Because encoding and decoding are started on the same + // end of the byte, we need to reverse `bytes_to_store_sizes` + // to get sizes in order. + bytes_to_store_sizes.reverse(); - size_lens + bytes_to_store_sizes } impl Storable for (A, B, C) @@ -678,28 +687,37 @@ where fn to_bytes(&self) -> Cow<[u8]> { let a_bytes = self.0.to_bytes(); let a_size = a_bytes.len(); - let a_size_len = size_len(a_size); + let b_bytes = self.1.to_bytes(); let b_size = b_bytes.len(); - let b_size_len = size_len(b_size); + let c_bytes = self.2.to_bytes(); let c_size = c_bytes.len(); - let output_size = a_size + b_size + c_size + a_size_len + b_size_len + 1; + let output_size = a_size + + b_size + + c_size + + bytes_to_store_size(a_size) + + bytes_to_store_size(b_size) + + 1; let mut bytes = vec![0; output_size]; + let mut curr_ind = 0; - bytes[0] = get_size_len_byte(vec![a_size_len, b_size_len]); + + bytes[0] = encode_size_lengths(vec![a_size, b_size]); curr_ind += 1; - encode_size(&mut bytes[curr_ind..], a_size, a_size_len); - curr_ind += a_size_len; + + curr_ind += encode_size(&mut bytes[curr_ind..], a_size); bytes[curr_ind..curr_ind + a_size].copy_from_slice(a_bytes.borrow()); curr_ind += a_size; - encode_size(&mut bytes[curr_ind..], b_size, b_size_len); - curr_ind += b_size_len; + + curr_ind += encode_size(&mut bytes[curr_ind..], b_size); bytes[curr_ind..curr_ind + b_size].copy_from_slice(b_bytes.borrow()); curr_ind += b_size; + bytes[curr_ind..curr_ind + c_size].copy_from_slice(c_bytes.borrow()); + debug_assert_eq!(curr_ind + c_size, output_size); Cow::Owned(bytes) @@ -707,7 +725,7 @@ where fn from_bytes(bytes: Cow<[u8]>) -> Self { let mut curr_ind = 0; - let sizes_len = get_size_lens(bytes[0], 3); + let sizes_len = decode_sizes_length(bytes[0], 2); curr_ind += 1; let a_size_len = sizes_len[0] as usize; let a_size = decode_size(&bytes[curr_ind..], a_size_len); @@ -733,11 +751,11 @@ where Bound::Bounded { max_size: a_bounds.max_size - + bytes_to_store_size(&a_bounds) + + bytes_to_store_size_bounded(&a_bounds) + b_bounds.max_size - + bytes_to_store_size(&b_bounds) + + bytes_to_store_size_bounded(&b_bounds) + c_bounds.max_size - + bytes_to_store_size(&c_bounds), + + bytes_to_store_size_bounded(&c_bounds), is_fixed_size: a_bounds.is_fixed_size && b_bounds.is_fixed_size && c_bounds.is_fixed_size, From f063da07b471816debb8ecf35d4846e3faa920c2 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 6 Mar 2024 13:03:49 +0000 Subject: [PATCH 05/17] add comments --- src/storable.rs | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 61db2377..d58f4ddf 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -597,6 +597,7 @@ fn encode_size_of_bound(dst: &mut [u8], n: usize, bounds: &Bounds) { } } +/// Decodes size from the beginning of `src` of length `size_len` and returns it. fn decode_size(src: &[u8], size_len: usize) -> usize { match size_len { 1 => src[0] as usize, @@ -605,6 +606,7 @@ fn decode_size(src: &[u8], size_len: usize) -> usize { } } +/// Encodes size at beginning of `dst` and returns the length of the encoding. fn encode_size(dst: &mut [u8], n: usize) -> usize { let bytes_to_store_size = bytes_to_store_size(n); match bytes_to_store_size { @@ -651,12 +653,12 @@ fn encode_size_lengths(sizes: Vec) -> u8 { size_lengths_byte } -fn decode_sizes_length(mut encoded_bytes_to_store: u8, number_of_encoded_lengths: u8) -> Vec { +fn decode_size_lengths(mut encoded_bytes_to_store: u8, number_of_encoded_lengths: u8) -> Vec { assert!(number_of_encoded_lengths <= 4); let mut bytes_to_store_sizes = vec![]; - for _ in 0..number_of_encoded_lengths - 1 { + for _ in 0..number_of_encoded_lengths { // The number of bytes required to store the size of every // element is represented with 2 bits. Hence we use // mask `11`, equivalent to 3 in the decimal system. @@ -684,6 +686,8 @@ where B: Storable, C: Storable, { + // Touple will be serialized in the following form: + // fn to_bytes(&self) -> Cow<[u8]> { let a_bytes = self.0.to_bytes(); let a_size = a_bytes.len(); @@ -694,19 +698,18 @@ where let c_bytes = self.2.to_bytes(); let c_size = c_bytes.len(); - let output_size = a_size - + b_size - + c_size + let output_size = 1 + bytes_to_store_size(a_size) + + a_size + bytes_to_store_size(b_size) - + 1; + + b_size + + c_size; let mut bytes = vec![0; output_size]; - let mut curr_ind = 0; - bytes[0] = encode_size_lengths(vec![a_size, b_size]); - curr_ind += 1; + + let mut curr_ind = 1; curr_ind += encode_size(&mut bytes[curr_ind..], a_size); bytes[curr_ind..curr_ind + a_size].copy_from_slice(a_bytes.borrow()); @@ -724,19 +727,22 @@ where } fn from_bytes(bytes: Cow<[u8]>) -> Self { - let mut curr_ind = 0; - let sizes_len = decode_sizes_length(bytes[0], 2); - curr_ind += 1; - let a_size_len = sizes_len[0] as usize; + let size_lengths = decode_size_lengths(bytes[0], 2); + + let mut curr_ind = 1; + + let a_size_len = size_lengths[0] as usize; let a_size = decode_size(&bytes[curr_ind..], a_size_len); curr_ind += a_size_len; let a = A::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + a_size])); curr_ind += a_size; - let b_size_len = sizes_len[1] as usize; + + let b_size_len = size_lengths[1] as usize; let b_size = decode_size(&bytes[curr_ind..], b_size_len); curr_ind += b_size_len; let b = B::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + b_size])); curr_ind += b_size; + let c = C::from_bytes(Cow::Borrowed(&bytes[curr_ind..])); (a, b, c) From 2870baf2ceaf180ed4dbeb4bc25d8277057bf16b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 6 Mar 2024 13:08:14 +0000 Subject: [PATCH 06/17] refactor --- src/storable.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index d58f4ddf..f7061ff7 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -574,12 +574,8 @@ pub(crate) const fn bounds() -> Bounds { fn decode_size_of_bound(src: &[u8], bounds: &Bounds) -> usize { if bounds.is_fixed_size { bounds.max_size as usize - } else if bounds.max_size <= u8::MAX as u32 { - src[0] as usize - } else if bounds.max_size <= u16::MAX as u32 { - u16::from_be_bytes([src[0], src[1]]) as usize } else { - u32::from_be_bytes([src[0], src[1], src[2], src[3]]) as usize + decode_size(src, bytes_to_store_size(bounds.max_size as usize)) } } From 96b017e1ea8dce41fff5d60487a2f5d0808cbade Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 6 Mar 2024 13:14:09 +0000 Subject: [PATCH 07/17] refactor --- src/storable.rs | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index f7061ff7..965f0e6b 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -583,14 +583,7 @@ fn encode_size_of_bound(dst: &mut [u8], n: usize, bounds: &Bounds) { if bounds.is_fixed_size { return; } - - if bounds.max_size <= u8::MAX as u32 { - dst[0] = n as u8; - } else if bounds.max_size <= u16::MAX as u32 { - dst[0..2].copy_from_slice(&(n as u16).to_be_bytes()); - } else { - dst[0..4].copy_from_slice(&(n as u32).to_be_bytes()); - } + encode_size(dst, n, bytes_to_store_size(bounds.max_size as usize)); } /// Decodes size from the beginning of `src` of length `size_len` and returns it. @@ -602,15 +595,13 @@ fn decode_size(src: &[u8], size_len: usize) -> usize { } } -/// Encodes size at beginning of `dst` and returns the length of the encoding. -fn encode_size(dst: &mut [u8], n: usize) -> usize { - let bytes_to_store_size = bytes_to_store_size(n); +/// Encodes `size` at the beginning of `dst` of length `bytes_to_store_size` bytes. +fn encode_size(dst: &mut [u8], size: usize, bytes_to_store_size: usize) { match bytes_to_store_size { - 1 => dst[0] = n as u8, - 2 => dst[0..2].copy_from_slice(&(n as u16).to_be_bytes()), - _ => dst[0..4].copy_from_slice(&(n as u32).to_be_bytes()), + 1 => dst[0] = size as u8, + 2 => dst[0..2].copy_from_slice(&(size as u16).to_be_bytes()), + _ => dst[0..4].copy_from_slice(&(size as u32).to_be_bytes()), }; - bytes_to_store_size } pub(crate) const fn bytes_to_store_size_bounded(bounds: &Bounds) -> u32 { @@ -707,11 +698,13 @@ where let mut curr_ind = 1; - curr_ind += encode_size(&mut bytes[curr_ind..], a_size); + encode_size(&mut bytes[curr_ind..], a_size, bytes_to_store_size(a_size)); + curr_ind += bytes_to_store_size(a_size); bytes[curr_ind..curr_ind + a_size].copy_from_slice(a_bytes.borrow()); curr_ind += a_size; - curr_ind += encode_size(&mut bytes[curr_ind..], b_size); + encode_size(&mut bytes[curr_ind..], b_size, bytes_to_store_size(b_size)); + curr_ind += bytes_to_store_size(b_size); bytes[curr_ind..curr_ind + b_size].copy_from_slice(b_bytes.borrow()); curr_ind += b_size; From b6464b24cb7df839c12333db7c3eae10d68ad7bf Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Thu, 7 Mar 2024 16:56:19 +0100 Subject: [PATCH 08/17] Update src/storable.rs Co-authored-by: Islam El-Ashi --- src/storable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storable.rs b/src/storable.rs index 965f0e6b..c0d7f451 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -673,7 +673,7 @@ where B: Storable, C: Storable, { - // Touple will be serialized in the following form: + // Tuple will be serialized in the following form: // fn to_bytes(&self) -> Cow<[u8]> { let a_bytes = self.0.to_bytes(); From 30d0f78b0623342193754dbedb02563b3d2280bb Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Fri, 8 Mar 2024 12:13:27 +0000 Subject: [PATCH 09/17] Adress comments --- src/storable.rs | 117 +++++++++++++++++++++++++++++++----------- src/storable/tests.rs | 16 +++--- 2 files changed, 97 insertions(+), 36 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index c0d7f451..c630d1e0 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -667,14 +667,42 @@ fn decode_size_lengths(mut encoded_bytes_to_store: u8, number_of_encoded_lengths bytes_to_store_sizes } +const fn get_fixed_sizes() -> Option<(usize, usize)> +where + A: Storable, + B: Storable, +{ + if let Bound::Bounded { + max_size: max_size_a, + is_fixed_size: is_fixed_size_a, + } = A::BOUND + { + if is_fixed_size_a { + if let Bound::Bounded { + max_size: max_size_b, + is_fixed_size: is_fixed_size_b, + } = B::BOUND + { + if is_fixed_size_b { + return Some((max_size_a as usize, max_size_b as usize)); + } + } + } + } + None +} + impl Storable for (A, B, C) where A: Storable, B: Storable, C: Storable, { - // Tuple will be serialized in the following form: - // + // Tuple (A, B, C) will be serialized in the following form: + // If A and B have fixed size + // + // Otherwise + // fn to_bytes(&self) -> Cow<[u8]> { let a_bytes = self.0.to_bytes(); let a_size = a_bytes.len(); @@ -685,26 +713,35 @@ where let c_bytes = self.2.to_bytes(); let c_size = c_bytes.len(); - let output_size = 1 - + bytes_to_store_size(a_size) - + a_size - + bytes_to_store_size(b_size) - + b_size - + c_size; + let a_b_fixed_sizes = get_fixed_sizes::(); - let mut bytes = vec![0; output_size]; + let sizes_overhead = if a_b_fixed_sizes.is_some() { + 0 + } else { + 1 + bytes_to_store_size(a_size) + bytes_to_store_size(b_size) + }; - bytes[0] = encode_size_lengths(vec![a_size, b_size]); + let output_size = a_size + b_size + c_size + sizes_overhead; - let mut curr_ind = 1; + let mut curr_ind = 0; + + let mut bytes = vec![0; output_size]; + + if a_b_fixed_sizes.is_none() { + bytes[curr_ind] = encode_size_lengths(vec![a_size, b_size]); + curr_ind += 1; + encode_size(&mut bytes[curr_ind..], a_size, bytes_to_store_size(a_size)); + curr_ind += bytes_to_store_size(a_size); + } - encode_size(&mut bytes[curr_ind..], a_size, bytes_to_store_size(a_size)); - curr_ind += bytes_to_store_size(a_size); bytes[curr_ind..curr_ind + a_size].copy_from_slice(a_bytes.borrow()); curr_ind += a_size; - encode_size(&mut bytes[curr_ind..], b_size, bytes_to_store_size(b_size)); - curr_ind += bytes_to_store_size(b_size); + if a_b_fixed_sizes.is_none() { + encode_size(&mut bytes[curr_ind..], b_size, bytes_to_store_size(b_size)); + curr_ind += bytes_to_store_size(b_size); + } + bytes[curr_ind..curr_ind + b_size].copy_from_slice(b_bytes.borrow()); curr_ind += b_size; @@ -716,21 +753,36 @@ where } fn from_bytes(bytes: Cow<[u8]>) -> Self { - let size_lengths = decode_size_lengths(bytes[0], 2); + let a_b_fixed_sizes = get_fixed_sizes::(); - let mut curr_ind = 1; + let mut curr_ind = 0; - let a_size_len = size_lengths[0] as usize; - let a_size = decode_size(&bytes[curr_ind..], a_size_len); - curr_ind += a_size_len; - let a = A::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + a_size])); - curr_ind += a_size; + let (a, b) = if let Some((a_size, b_size)) = a_b_fixed_sizes { + let a = A::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + a_size])); + curr_ind += a_size; - let b_size_len = size_lengths[1] as usize; - let b_size = decode_size(&bytes[curr_ind..], b_size_len); - curr_ind += b_size_len; - let b = B::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + b_size])); - curr_ind += b_size; + let b = B::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + b_size])); + curr_ind += b_size; + + (a, b) + } else { + let size_lengths = decode_size_lengths(bytes[curr_ind], 2); + curr_ind += 1; + + let a_size_len = size_lengths[0] as usize; + let a_size = decode_size(&bytes[curr_ind..], a_size_len); + curr_ind += a_size_len; + let a = A::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + a_size])); + curr_ind += a_size; + + let b_size_len = size_lengths[1] as usize; + let b_size = decode_size(&bytes[curr_ind..], b_size_len); + curr_ind += b_size_len; + let b = B::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + b_size])); + curr_ind += b_size; + + (a, b) + }; let c = C::from_bytes(Cow::Borrowed(&bytes[curr_ind..])); @@ -744,13 +796,18 @@ where let b_bounds = bounds::(); let c_bounds = bounds::(); + let sizes_overhead = if a_bounds.is_fixed_size && b_bounds.is_fixed_size { + 0 + } else { + 1 + bytes_to_store_size_bounded(&a_bounds) + + bytes_to_store_size_bounded(&b_bounds) + }; + Bound::Bounded { max_size: a_bounds.max_size - + bytes_to_store_size_bounded(&a_bounds) + b_bounds.max_size - + bytes_to_store_size_bounded(&b_bounds) + c_bounds.max_size - + bytes_to_store_size_bounded(&c_bounds), + + sizes_overhead, is_fixed_size: a_bounds.is_fixed_size && b_bounds.is_fixed_size && c_bounds.is_fixed_size, diff --git a/src/storable/tests.rs b/src/storable/tests.rs index 2e12426e..ae9f3091 100644 --- a/src/storable/tests.rs +++ b/src/storable/tests.rs @@ -14,11 +14,11 @@ proptest! { } #[test] - fn tuple_with_three_elements_roundtrip(x in any::(), y in uniform20(any::()), z in uniform20(any::())) { + fn tuple_with_three_elements_fixed_size_roundtrip(x in any::(), y in uniform20(any::()), z in uniform20(any::())) { let tuple = (x, y, z); let bytes = tuple.to_bytes(); - // 1B sizes len | 1B x size | 8B x bytes | 1B y size | 20B y bytes | 20B z bytes - prop_assert_eq!(bytes.len(), 51); + // 8B x bytes | 20B y bytes | 20B z bytes + prop_assert_eq!(bytes.len(), 48); prop_assert_eq!(tuple, Storable::from_bytes(bytes)); } @@ -29,9 +29,13 @@ proptest! { } #[test] - fn tuple_with_three_elements_bounded_and_unbounded_roundtrip(v1 in pvec(any::(), 0..4), x in any::(), v2 in pvec(any::(), 0..12)) { - let tuple = (v1, x, v2); - assert_eq!(tuple, Storable::from_bytes(tuple.to_bytes())); + fn tuple_with_three_elements_bounded_and_unbounded_roundtrip(x in pvec(any::(), 4..5), y in any::(), z in pvec(any::(), 12..13)) { + let tuple = (x, y, z); + let tuple_copy = tuple.clone(); + let bytes = tuple_copy.to_bytes(); + // 1B sizes len | 1B x size | 4B x bytes | 1B y size | 8B y bytes | 12B z bytes + prop_assert_eq!(bytes.len(), 27); + prop_assert_eq!(tuple, Storable::from_bytes(bytes)); } From 0628024dc8734717ac6ca34b9e0a7e75cee0e71b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Fri, 8 Mar 2024 12:20:05 +0000 Subject: [PATCH 10/17] refactor --- src/storable.rs | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index c630d1e0..869c8ab3 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -672,24 +672,19 @@ where A: Storable, B: Storable, { - if let Bound::Bounded { - max_size: max_size_a, - is_fixed_size: is_fixed_size_a, - } = A::BOUND - { - if is_fixed_size_a { - if let Bound::Bounded { + match (A::BOUND, B::BOUND) { + ( + Bound::Bounded { + max_size: max_size_a, + is_fixed_size: true, + }, + Bound::Bounded { max_size: max_size_b, - is_fixed_size: is_fixed_size_b, - } = B::BOUND - { - if is_fixed_size_b { - return Some((max_size_a as usize, max_size_b as usize)); - } - } - } + is_fixed_size: true, + }, + ) => Some((max_size_a as usize, max_size_b as usize)), + _ => None, } - None } impl Storable for (A, B, C) From 360e053825814dbeb015a614576b72a7e78c41cb Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 12 Mar 2024 14:51:25 +0000 Subject: [PATCH 11/17] refactor --- src/storable.rs | 190 +++++++++++++++++++++++++----------------- src/storable/tests.rs | 4 +- 2 files changed, 115 insertions(+), 79 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 869c8ab3..a0f4ece7 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -667,24 +667,72 @@ fn decode_size_lengths(mut encoded_bytes_to_store: u8, number_of_encoded_lengths bytes_to_store_sizes } -const fn get_fixed_sizes() -> Option<(usize, usize)> -where - A: Storable, - B: Storable, -{ - match (A::BOUND, B::BOUND) { - ( - Bound::Bounded { - max_size: max_size_a, - is_fixed_size: true, - }, - Bound::Bounded { - max_size: max_size_b, - is_fixed_size: true, - }, - ) => Some((max_size_a as usize, max_size_b as usize)), - _ => None, +// Encodes a serialized element `T` in a tuple. +// The element is assumed to be at the beginning of `dst`. +// Returns the number of bytes written to `dst`. +fn encode_tuple_element(dst: &mut [u8], bytes: &[u8], last: bool) -> usize { + let mut bytes_written: usize = 0; + let size = bytes.len(); + + if !last && !T::BOUND.is_fixed_size() { + encode_size(&mut dst[bytes_written..], size, bytes_to_store_size(size)); + bytes_written += bytes_to_store_size(size); } + + dst[bytes_written..bytes_written + size].copy_from_slice(bytes); + bytes_written + size +} + +// Decodes an element `T` from a tuple. +// +// The element is assumed to be at the beginning of `src`. +// The length of the size of the element should be provided if the element is *not* fixed in size. +// +// Returns the element `T` and the number of bytes read from `src`. +fn decode_tuple_element(src: &[u8], size_len: Option, last: bool) -> (T, usize) { + let mut bytes_read: usize = 0; + + let size = if last { + src.len() + } else { + if let Bound::Bounded { + max_size, + is_fixed_size: true, + } = T::BOUND + { + max_size as usize + } else { + let size_len = size_len.unwrap() as usize; + let size = decode_size(&src[bytes_read..], size_len); + bytes_read += size_len; + size + } + }; + + ( + T::from_bytes(Cow::Borrowed(&src[bytes_read..bytes_read + size])), + bytes_read + size, + ) +} + +// Returns number of bytes required to store encoding of size for A and B. +const fn sizes_overhead(a_size: usize, b_size: usize) -> usize { + let mut sizes_overhead = 0; + + if !(A::BOUND.is_fixed_size() && B::BOUND.is_fixed_size()) { + // 1B for size lengths encoding + sizes_overhead += 1; + + if !A::BOUND.is_fixed_size() { + sizes_overhead += bytes_to_store_size(a_size); + } + + if !B::BOUND.is_fixed_size() { + sizes_overhead += bytes_to_store_size(b_size); + } + } + + sizes_overhead } impl Storable for (A, B, C) @@ -697,7 +745,7 @@ where // If A and B have fixed size // // Otherwise - // + // fn to_bytes(&self) -> Cow<[u8]> { let a_bytes = self.0.to_bytes(); let a_size = a_bytes.len(); @@ -708,78 +756,69 @@ where let c_bytes = self.2.to_bytes(); let c_size = c_bytes.len(); - let a_b_fixed_sizes = get_fixed_sizes::(); - - let sizes_overhead = if a_b_fixed_sizes.is_some() { - 0 - } else { - 1 + bytes_to_store_size(a_size) + bytes_to_store_size(b_size) - }; + let sizes_overhead = sizes_overhead::(a_size, b_size); let output_size = a_size + b_size + c_size + sizes_overhead; - let mut curr_ind = 0; + let mut bytes_written = 0; let mut bytes = vec![0; output_size]; - if a_b_fixed_sizes.is_none() { - bytes[curr_ind] = encode_size_lengths(vec![a_size, b_size]); - curr_ind += 1; - encode_size(&mut bytes[curr_ind..], a_size, bytes_to_store_size(a_size)); - curr_ind += bytes_to_store_size(a_size); + if sizes_overhead != 0 { + bytes[bytes_written] = encode_size_lengths(vec![a_size, b_size]); + bytes_written += 1; } - bytes[curr_ind..curr_ind + a_size].copy_from_slice(a_bytes.borrow()); - curr_ind += a_size; + bytes_written += + encode_tuple_element::(&mut bytes[bytes_written..], a_bytes.borrow(), false); + bytes_written += + encode_tuple_element::(&mut bytes[bytes_written..], b_bytes.borrow(), false); + bytes_written += + encode_tuple_element::(&mut bytes[bytes_written..], c_bytes.borrow(), true); - if a_b_fixed_sizes.is_none() { - encode_size(&mut bytes[curr_ind..], b_size, bytes_to_store_size(b_size)); - curr_ind += bytes_to_store_size(b_size); - } - - bytes[curr_ind..curr_ind + b_size].copy_from_slice(b_bytes.borrow()); - curr_ind += b_size; - - bytes[curr_ind..curr_ind + c_size].copy_from_slice(c_bytes.borrow()); - - debug_assert_eq!(curr_ind + c_size, output_size); + assert_eq!(bytes_written, output_size); Cow::Owned(bytes) } fn from_bytes(bytes: Cow<[u8]>) -> Self { - let a_b_fixed_sizes = get_fixed_sizes::(); + let mut bytes_read_total = 0; - let mut curr_ind = 0; + let size_lengths = if A::BOUND.is_fixed_size() && B::BOUND.is_fixed_size() { + vec![] + } else { + let lengths = decode_size_lengths(bytes[bytes_read_total], 2); + bytes_read_total += 1; + lengths + }; - let (a, b) = if let Some((a_size, b_size)) = a_b_fixed_sizes { - let a = A::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + a_size])); - curr_ind += a_size; + let (a, bytes_read) = decode_tuple_element::( + &bytes[bytes_read_total..], + if A::BOUND.is_fixed_size() { + None + } else { + Some(size_lengths[0]) + }, + false, + ); + bytes_read_total += bytes_read; + + let (b, bytes_read) = decode_tuple_element::( + &bytes[bytes_read_total..], + if B::BOUND.is_fixed_size() { + None + } else { + Some(size_lengths[1]) + }, + false, + ); + bytes_read_total += bytes_read; - let b = B::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + b_size])); - curr_ind += b_size; + let (c, bytes_read) = decode_tuple_element::(&bytes[bytes_read_total..], None, true); - (a, b) - } else { - let size_lengths = decode_size_lengths(bytes[curr_ind], 2); - curr_ind += 1; - - let a_size_len = size_lengths[0] as usize; - let a_size = decode_size(&bytes[curr_ind..], a_size_len); - curr_ind += a_size_len; - let a = A::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + a_size])); - curr_ind += a_size; - - let b_size_len = size_lengths[1] as usize; - let b_size = decode_size(&bytes[curr_ind..], b_size_len); - curr_ind += b_size_len; - let b = B::from_bytes(Cow::Borrowed(&bytes[curr_ind..curr_ind + b_size])); - curr_ind += b_size; - - (a, b) - }; + bytes_read_total += bytes_read; - let c = C::from_bytes(Cow::Borrowed(&bytes[curr_ind..])); + assert_eq!(bytes_read_total, bytes.len()); (a, b, c) } @@ -791,12 +830,9 @@ where let b_bounds = bounds::(); let c_bounds = bounds::(); - let sizes_overhead = if a_bounds.is_fixed_size && b_bounds.is_fixed_size { - 0 - } else { - 1 + bytes_to_store_size_bounded(&a_bounds) - + bytes_to_store_size_bounded(&b_bounds) - }; + let sizes_overhead = + sizes_overhead::(a_bounds.max_size as usize, b_bounds.max_size as usize) + as u32; Bound::Bounded { max_size: a_bounds.max_size diff --git a/src/storable/tests.rs b/src/storable/tests.rs index ae9f3091..2b257776 100644 --- a/src/storable/tests.rs +++ b/src/storable/tests.rs @@ -33,8 +33,8 @@ proptest! { let tuple = (x, y, z); let tuple_copy = tuple.clone(); let bytes = tuple_copy.to_bytes(); - // 1B sizes len | 1B x size | 4B x bytes | 1B y size | 8B y bytes | 12B z bytes - prop_assert_eq!(bytes.len(), 27); + // 1B sizes len | 1B x size | 4B x bytes | 0B y size | 8B y bytes | 12B z bytes + prop_assert_eq!(bytes.len(), 26); prop_assert_eq!(tuple, Storable::from_bytes(bytes)); } From 8f8e326a41c6c3a45beaa917605ca86429fb049a Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 12 Mar 2024 14:55:00 +0000 Subject: [PATCH 12/17] clippy --- src/storable.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index a0f4ece7..d9d9efec 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -694,19 +694,17 @@ fn decode_tuple_element(src: &[u8], size_len: Option, last: boo let size = if last { src.len() + } else if let Bound::Bounded { + max_size, + is_fixed_size: true, + } = T::BOUND + { + max_size as usize } else { - if let Bound::Bounded { - max_size, - is_fixed_size: true, - } = T::BOUND - { - max_size as usize - } else { - let size_len = size_len.unwrap() as usize; - let size = decode_size(&src[bytes_read..], size_len); - bytes_read += size_len; - size - } + let size_len = size_len.unwrap() as usize; + let size = decode_size(&src[bytes_read..], size_len); + bytes_read += size_len; + size }; ( From 2c7daa0d0f5e40c851942b3fcb4bfd21e1886c00 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 12 Mar 2024 15:14:43 +0000 Subject: [PATCH 13/17] refactor --- src/storable.rs | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index d9d9efec..84913c9e 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -713,7 +713,7 @@ fn decode_tuple_element(src: &[u8], size_len: Option, last: boo ) } -// Returns number of bytes required to store encoding of size for A and B. +// Returns number of bytes required to store encoding of sizes for elements of type A and B. const fn sizes_overhead(a_size: usize, b_size: usize) -> usize { let mut sizes_overhead = 0; @@ -782,34 +782,27 @@ where fn from_bytes(bytes: Cow<[u8]>) -> Self { let mut bytes_read_total = 0; - let size_lengths = if A::BOUND.is_fixed_size() && B::BOUND.is_fixed_size() { - vec![] - } else { + let mut size_lengths = vec![None, None]; + + if !(A::BOUND.is_fixed_size() && B::BOUND.is_fixed_size()) { let lengths = decode_size_lengths(bytes[bytes_read_total], 2); bytes_read_total += 1; - lengths - }; - let (a, bytes_read) = decode_tuple_element::( - &bytes[bytes_read_total..], - if A::BOUND.is_fixed_size() { - None - } else { - Some(size_lengths[0]) - }, - false, - ); + if !A::BOUND.is_fixed_size() { + size_lengths[0] = Some(lengths[0]); + } + + if !B::BOUND.is_fixed_size() { + size_lengths[1] = Some(lengths[1]); + } + } + + let (a, bytes_read) = + decode_tuple_element::(&bytes[bytes_read_total..], size_lengths[0], false); bytes_read_total += bytes_read; - let (b, bytes_read) = decode_tuple_element::( - &bytes[bytes_read_total..], - if B::BOUND.is_fixed_size() { - None - } else { - Some(size_lengths[1]) - }, - false, - ); + let (b, bytes_read) = + decode_tuple_element::(&bytes[bytes_read_total..], size_lengths[1], false); bytes_read_total += bytes_read; let (c, bytes_read) = decode_tuple_element::(&bytes[bytes_read_total..], None, true); From ef111784df99e1108b1c780a3e34ebc2614e629e Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 12 Mar 2024 20:59:27 +0000 Subject: [PATCH 14/17] clippy --- src/storable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storable.rs b/src/storable.rs index 84913c9e..7c4113cc 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -782,7 +782,7 @@ where fn from_bytes(bytes: Cow<[u8]>) -> Self { let mut bytes_read_total = 0; - let mut size_lengths = vec![None, None]; + let mut size_lengths = [None, None]; if !(A::BOUND.is_fixed_size() && B::BOUND.is_fixed_size()) { let lengths = decode_size_lengths(bytes[bytes_read_total], 2); From 432058fb842d958b401bd684db92be43c30531aa Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 12 Mar 2024 21:12:02 +0000 Subject: [PATCH 15/17] refactor --- src/storable.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/storable.rs b/src/storable.rs index 7c4113cc..2b0b73cd 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -692,8 +692,10 @@ fn encode_tuple_element(dst: &mut [u8], bytes: &[u8], last: bool) - fn decode_tuple_element(src: &[u8], size_len: Option, last: bool) -> (T, usize) { let mut bytes_read: usize = 0; - let size = if last { - src.len() + let size = if let Some(size_len) = size_len { + let size = decode_size(&src[bytes_read..], size_len as usize); + bytes_read += size_len as usize; + size } else if let Bound::Bounded { max_size, is_fixed_size: true, @@ -701,10 +703,8 @@ fn decode_tuple_element(src: &[u8], size_len: Option, last: boo { max_size as usize } else { - let size_len = size_len.unwrap() as usize; - let size = decode_size(&src[bytes_read..], size_len); - bytes_read += size_len; - size + assert!(last); + src.len() }; ( From 989790df2ef712c5a7d53586eef4664911eac879 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 12 Mar 2024 21:12:25 +0000 Subject: [PATCH 16/17] . --- src/storable.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storable.rs b/src/storable.rs index 2b0b73cd..f79bf88c 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -703,6 +703,7 @@ fn decode_tuple_element(src: &[u8], size_len: Option, last: boo { max_size as usize } else { + // This case should only happen for the last element. assert!(last); src.len() }; From f5ba66ba7433e13d2ffc73ba53f53eda099b7cb0 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 13 Mar 2024 14:37:45 +0000 Subject: [PATCH 17/17] Add bound tests --- src/storable.rs | 1 + src/storable/tests.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/storable.rs b/src/storable.rs index f79bf88c..05fd42b2 100644 --- a/src/storable.rs +++ b/src/storable.rs @@ -50,6 +50,7 @@ pub trait Storable { } } +#[derive(Debug, PartialEq)] /// States whether the type's size is bounded or unbounded. pub enum Bound { /// The type has no size bounds. diff --git a/src/storable/tests.rs b/src/storable/tests.rs index 2b257776..a735defa 100644 --- a/src/storable/tests.rs +++ b/src/storable/tests.rs @@ -239,3 +239,46 @@ fn storable_for_bool() { assert!(!bool::from_bytes(false.to_bytes())); assert!(bool::from_bytes(true.to_bytes())); } + +#[test] +fn tuple_with_three_elements_test_bound() { + // <8B a_bytes> <8B b_bytes> <8B c_bytes> + assert_eq!(<(u64, u64, u64)>::BOUND.max_size(), 24); + assert!(<(u64, u64, u64)>::BOUND.is_fixed_size()); + + // <8B a_bytes> <8B b_bytes> <8B c_bytes> + assert_eq!(<(u64, u64, Blob<8>)>::BOUND.max_size(), 24); + assert!(!<(u64, u64, Blob<8>)>::BOUND.is_fixed_size()); + + // <1B size_lengths> <1B size_a> <8B a_bytes> <0B size_b> <8B b_bytes> <8B c_bytes> + assert_eq!(<(Blob<8>, u64, u64)>::BOUND.max_size(), 26); + assert!(!<(Blob<8>, u64, u64)>::BOUND.is_fixed_size()); + + // <1B size_lengths> <0B size_a> <8B a_bytes> <1B size_b> <8B b_bytes> <8B c_bytes> + assert_eq!(<(u64, Blob<8>, u64)>::BOUND.max_size(), 26); + assert!(!<(u64, Blob<8>, u64)>::BOUND.is_fixed_size()); + + // <1B size_lengths> <1B size_a> <8B a_bytes> <1B size_b> <8B b_bytes> <8B c_bytes> + assert_eq!(<(Blob<8>, Blob<8>, u64)>::BOUND.max_size(), 27); + assert!(!<(Blob<8>, Blob<8>, u64)>::BOUND.is_fixed_size()); + + // <1B size_lengths> <1B size_a> <8B a_bytes> <0B size_b> <8B b_bytes> <8B c_bytes> + assert_eq!(<(Blob<8>, u64, Blob<8>)>::BOUND.max_size(), 26); + assert!(!<(Blob<8>, u64, Blob<8>)>::BOUND.is_fixed_size()); + + // <1B size_lengths> <0B size_a> <8B a_bytes> <1B size_b> <8B b_bytes> <8B c_bytes> + assert_eq!(<(u64, Blob<8>, Blob<8>)>::BOUND.max_size(), 26); + assert!(!<(u64, Blob<8>, Blob<8>)>::BOUND.is_fixed_size()); + + // <1B size_lengths> <1B size_a> <8B a_bytes> <1B size_b> <8B b_bytes> <8B c_bytes> + assert_eq!(<(Blob<8>, Blob<8>, Blob<8>)>::BOUND.max_size(), 27); + assert!(!<(Blob<8>, Blob<8>, Blob<8>)>::BOUND.is_fixed_size()); + + assert_eq!(<(Blob<8>, Blob<8>, String)>::BOUND, Bound::Unbounded); + assert_eq!(<(Blob<8>, String, Blob<8>)>::BOUND, Bound::Unbounded); + assert_eq!(<(String, Blob<8>, Blob<8>)>::BOUND, Bound::Unbounded); + assert_eq!(<(String, String, Blob<8>)>::BOUND, Bound::Unbounded); + assert_eq!(<(String, Blob<8>, String)>::BOUND, Bound::Unbounded); + assert_eq!(<(Blob<8>, String, String)>::BOUND, Bound::Unbounded); + assert_eq!(<(String, String, String)>::BOUND, Bound::Unbounded); +}