From ea46926d7e1fadd18fa71f2141602876ff7bf1dc Mon Sep 17 00:00:00 2001 From: Mark Juggurnauth-Thomas Date: Wed, 26 Mar 2025 15:12:32 +0000 Subject: [PATCH 1/4] Rename bincode dev-dependency to bincode1 We want to add an optional dependency on bincode 2. The MSRV of bincode 2 is too high for us to use it in the tests, so rename the test usage to bincode1 in preparation. --- Cargo.toml | 2 +- src/tests.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 829b0c9..fe0e769 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ malloc_size_of = { version = "0.1", optional = true, default-features = false } arbitrary = { version = "1", optional = true } [dev-dependencies] -bincode = "1.0.1" +bincode1 = { package = "bincode", version = "1.0.1" } debugger_test = "0.1.0" debugger_test_parser = "0.1.0" diff --git a/src/tests.rs b/src/tests.rs index 9ccaf32..524efc9 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -835,7 +835,7 @@ fn test_write() { #[cfg(feature = "serde")] #[test] fn test_serde() { - use bincode::{config, deserialize}; + use bincode1::{config, deserialize}; let mut small_vec: SmallVec<[i32; 2]> = SmallVec::new(); small_vec.push(1); let encoded = config().limit(100).serialize(&small_vec).unwrap(); From 655cb0d036c6d1879c8aeeefe300a72d28be0ad3 Mon Sep 17 00:00:00 2001 From: Mark Juggurnauth-Thomas Date: Wed, 26 Mar 2025 15:34:08 +0000 Subject: [PATCH 2/4] Add support for bincode encode and decode Add implementations for `bincode`'s `Encode`, `Decode` and `BorrowDecode`. Like `bincode` itself, we implement optimizations when the vector item is `u8` so that `SmallVec<[u8; N]>` can be encoded and decoded with just a memory copy. --- Cargo.toml | 3 ++ src/lib.rs | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/tests.rs | 62 +++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index fe0e769..be92f71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ specialization = [] may_dangle = [] drain_filter = [] drain_keep_rest = ["drain_filter"] +impl_bincode = ["bincode", "unty"] # UNSTABLE FEATURES (requires Rust nightly) # Enable to use the #[debugger_visualizer] attribute. @@ -29,6 +30,8 @@ debugger_visualizer = [] serde = { version = "1", optional = true, default-features = false } malloc_size_of = { version = "0.1", optional = true, default-features = false } arbitrary = { version = "1", optional = true } +bincode = { version = "2", optional = true, default-features = false } +unty = { version = "0.0.4", optional = true, default-features = false } [dev-dependencies] bincode1 = { package = "bincode", version = "1.0.1" } diff --git a/src/lib.rs b/src/lib.rs index eca88d8..8689b75 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2498,3 +2498,106 @@ impl Clone for ConstNonNull { } impl Copy for ConstNonNull {} + +#[cfg(feature = "impl_bincode")] +use bincode::{ + de::{BorrowDecoder, Decode, Decoder, read::Reader}, + enc::{Encode, Encoder, write::Writer}, + error::{DecodeError, EncodeError}, + BorrowDecode, +}; + +#[cfg(feature = "impl_bincode")] +impl Decode for SmallVec +where + A: Array, + A::Item: Decode, +{ + fn decode>(decoder: &mut D) -> Result { + use core::convert::TryInto; + let len = u64::decode(decoder)?; + let len = len.try_into().map_err(|_| DecodeError::OutsideUsizeRange(len))?; + decoder.claim_container_read::(len)?; + + let mut vec = SmallVec::with_capacity(len); + if unty::type_equal::() { + // Initialize the smallvec's buffer. Note that we need to do this through + // the raw pointer as we cannot name the type [u8; N] even though A::Item is u8. + let ptr = vec.as_mut_ptr(); + // SAFETY: A::Item is u8 and the smallvec has been allocated with enough capacity + unsafe { + core::ptr::write_bytes(ptr, 0, len); + vec.set_len(len); + } + // Read the data into the smallvec's buffer. + let slice = vec.as_mut_slice(); + // SAFETY: A::Item is u8 + let slice = unsafe { core::mem::transmute::<&mut [A::Item], &mut [u8]>(slice) }; + decoder.reader().read(slice)?; + } else { + for _ in 0..len { + decoder.unclaim_bytes_read(core::mem::size_of::()); + vec.push(A::Item::decode(decoder)?); + } + } + Ok(vec) + } +} + +#[cfg(feature = "impl_bincode")] +impl<'de, A, Context> BorrowDecode<'de, Context> for SmallVec +where + A: Array, + A::Item: BorrowDecode<'de, Context>, +{ + fn borrow_decode>(decoder: &mut D) -> Result { + use core::convert::TryInto; + let len = u64::decode(decoder)?; + let len = len.try_into().map_err(|_| DecodeError::OutsideUsizeRange(len))?; + decoder.claim_container_read::(len)?; + + let mut vec = SmallVec::with_capacity(len); + if unty::type_equal::() { + // Initialize the smallvec's buffer. Note that we need to do this through + // the raw pointer as we cannot name the type [u8; N] even though A::Item is u8. + let ptr = vec.as_mut_ptr(); + // SAFETY: A::Item is u8 and the smallvec has been allocated with enough capacity + unsafe { + core::ptr::write_bytes(ptr, 0, len); + vec.set_len(len); + } + // Read the data into the smallvec's buffer. + let slice = vec.as_mut_slice(); + // SAFETY: A::Item is u8 + let slice = unsafe { core::mem::transmute::<&mut [A::Item], &mut [u8]>(slice) }; + decoder.reader().read(slice)?; + } else { + for _ in 0..len { + decoder.unclaim_bytes_read(core::mem::size_of::()); + vec.push(A::Item::borrow_decode(decoder)?); + } + } + Ok(vec) + } +} + +#[cfg(feature = "impl_bincode")] +impl Encode for SmallVec +where + A: Array, + A::Item: Encode, +{ + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + (self.len() as u64).encode(encoder)?; + if unty::type_equal::() { + // Safety: A::Item is u8 + let slice: &[u8] = unsafe { core::mem::transmute(self.as_slice()) }; + encoder.writer().write(slice)?; + } else { + for item in self.iter() { + item.encode(encoder)?; + } + } + Ok(()) + } +} diff --git a/src/tests.rs b/src/tests.rs index 524efc9..88c5f32 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1072,3 +1072,65 @@ fn test_insert_out_of_bounds() { let mut v: SmallVec<[i32; 4]> = SmallVec::new(); v.insert(10, 6); } + +#[cfg(feature = "impl_bincode")] +#[test] +fn test_bincode() { + let config = bincode::config::standard(); + let mut small_vec: SmallVec<[i32; 2]> = SmallVec::new(); + let mut buffer = [0u8; 128]; + small_vec.push(1); + let bytes_written = bincode::encode_into_slice(&small_vec, &mut buffer, config).unwrap(); + let (decoded, bytes_read) = + bincode::decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); + let (decoded, bytes_read) = + bincode::borrow_decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); + // Spill the vec + small_vec.push(2); + small_vec.push(3); + small_vec.push(4); + // Check again after spilling. + let bytes_written = bincode::encode_into_slice(&small_vec, &mut buffer, config).unwrap(); + let (decoded, bytes_read) = + bincode::decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); + let (decoded, bytes_read) = + bincode::borrow_decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); +} + +#[cfg(feature = "impl_bincode")] +#[test] +fn test_bincode_u8() { + let config = bincode::config::standard(); + let mut small_vec: SmallVec<[u8; 16]> = SmallVec::new(); + let mut buffer = [0u8; 128]; + small_vec.extend_from_slice(b"testing test"); + let bytes_written = bincode::encode_into_slice(&small_vec, &mut buffer, config).unwrap(); + let (decoded, bytes_read) = + bincode::decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); + let (decoded, bytes_read) = + bincode::borrow_decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); + // Spill the vec + small_vec.extend_from_slice(b"some more testing"); + // Check again after spilling. + let bytes_written = bincode::encode_into_slice(&small_vec, &mut buffer, config).unwrap(); + let (decoded, bytes_read) = + bincode::decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); + let (decoded, bytes_read) = + bincode::borrow_decode_from_slice::, _>(&buffer, config).unwrap(); + assert_eq!(bytes_written, bytes_read); + assert_eq!(small_vec, decoded); +} From f4de2c43a1734d632297da5c1d26a7b04781646d Mon Sep 17 00:00:00 2001 From: Matt Brubeck Date: Fri, 4 Apr 2025 17:34:01 -0700 Subject: [PATCH 3/4] Pin honggfuzz binary version --- fuzz/travis-fuzz.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/travis-fuzz.sh b/fuzz/travis-fuzz.sh index ff44d25..5782ea7 100755 --- a/fuzz/travis-fuzz.sh +++ b/fuzz/travis-fuzz.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -cargo install --force honggfuzz --version "^0.5.47" +cargo install --force honggfuzz --version 0.5.47 for TARGET in fuzz_targets/*; do FILENAME=$(basename $TARGET) FILE="${FILENAME%.*}" From 4add68e071d610e17d50ba454c5c34a4c141515b Mon Sep 17 00:00:00 2001 From: Matt Brubeck Date: Fri, 4 Apr 2025 17:41:25 -0700 Subject: [PATCH 4/4] Temporarily disable broken fuzzing in CI --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 317da29..e2e83f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,9 +17,9 @@ jobs: os: [ubuntu-latest] include: - toolchain: stable - fuzz: 1 + fuzz: 0 - toolchain: beta - fuzz: 1 + fuzz: 0 - os: windows-latest toolchain: nightly