feat(encoding): add dictionary compression support

zstd/Cargo.toml

@@ -11,7 +11,7 @@ license = "Apache-2.0"
 homepage = "https://github.com/structured-world/structured-zstd"
 repository = "https://github.com/structured-world/structured-zstd"
 description = "Pure Rust zstd implementation — managed fork of ruzstd. Dictionary decompression, no FFI."
-exclude = ["dict_tests/*", "fuzz_decodecorpus/*", "decodecorpus_files/*"]
+exclude = ["fuzz_decodecorpus/*", "decodecorpus_files/*", "dict_tests/files/**"]
 # Package metadata points at a crate-local symlink so the packaged crate and repo root README stay in sync.
 readme = "README.md"
 keywords = ["zstd", "zstandard", "decompression", "compression", "pure-rust"]

zstd/src/decoding/dictionary.rs

@@ -40,15 +40,50 @@ pub struct Dictionary {
 pub const MAGIC_NUM: [u8; 4] = [0x37, 0xA4, 0x30, 0xEC];
 
 impl Dictionary {
-    /// Parses the dictionary from `raw` and set the tables
-    /// it returns the dict_id for checking with the frame's `dict_id``
+    /// Build a dictionary from raw content bytes (without entropy table sections).
+    ///
+    /// This is primarily intended for dictionaries produced by the `dict_builder`
+    /// module, which currently emits raw-content dictionaries.
+    pub fn from_raw_content(
+        id: u32,
+        dict_content: Vec<u8>,
+    ) -> Result<Dictionary, DictionaryDecodeError> {
+        if id == 0 {
+            return Err(DictionaryDecodeError::ZeroDictionaryId);
+        }
+        if dict_content.is_empty() {
+            return Err(DictionaryDecodeError::DictionaryTooSmall { got: 0, need: 1 });
+        }
+
+        Ok(Dictionary {
+            id,
+            fse: FSEScratch::new(),
+            huf: HuffmanScratch::new(),
+            dict_content,
+            offset_hist: [1, 4, 8],
+        })
+    }
+
+    /// Parses the dictionary from `raw`, initializes its tables,
+    /// and returns a fully constructed [`Dictionary`] whose `id` can be
+    /// checked against the frame's `dict_id`.
     pub fn decode_dict(raw: &[u8]) -> Result<Dictionary, DictionaryDecodeError> {
+        const MIN_MAGIC_AND_ID_LEN: usize = 8;
+        const OFFSET_HISTORY_LEN: usize = 12;
+
+        if raw.len() < MIN_MAGIC_AND_ID_LEN {
+            return Err(DictionaryDecodeError::DictionaryTooSmall {
+                got: raw.len(),
+                need: MIN_MAGIC_AND_ID_LEN,
+            });
+        }
+
         let mut new_dict = Dictionary {
             id: 0,
             fse: FSEScratch::new(),
             huf: HuffmanScratch::new(),
             dict_content: Vec::new(),
-            offset_hist: [2, 4, 8],
+            offset_hist: [1, 4, 8],
         };
 
         let magic_num: [u8; 4] = raw[..4].try_into().expect("optimized away");
@@ -58,6 +93,9 @@ impl Dictionary {
 
         let dict_id = raw[4..8].try_into().expect("optimized away");
         let dict_id = u32::from_le_bytes(dict_id);
+        if dict_id == 0 {
+            return Err(DictionaryDecodeError::ZeroDictionaryId);
+        }
         new_dict.id = dict_id;
 
         let raw_tables = &raw[8..];
@@ -83,6 +121,13 @@ impl Dictionary {
         )?;
         let raw_tables = &raw_tables[ll_size..];
 
+        if raw_tables.len() < OFFSET_HISTORY_LEN {
+            return Err(DictionaryDecodeError::DictionaryTooSmall {
+                got: raw_tables.len(),
+                need: OFFSET_HISTORY_LEN,
+            });
+        }
+
         let offset1 = raw_tables[0..4].try_into().expect("optimized away");
         let offset1 = u32::from_le_bytes(offset1);
 
@@ -92,6 +137,16 @@ impl Dictionary {
         let offset3 = raw_tables[8..12].try_into().expect("optimized away");
         let offset3 = u32::from_le_bytes(offset3);
 
+        if offset1 == 0 {
+            return Err(DictionaryDecodeError::ZeroRepeatOffsetInDictionary { index: 0 });
+        }
+        if offset2 == 0 {
+            return Err(DictionaryDecodeError::ZeroRepeatOffsetInDictionary { index: 1 });
+        }
+        if offset3 == 0 {
+            return Err(DictionaryDecodeError::ZeroRepeatOffsetInDictionary { index: 2 });
+        }
+
         new_dict.offset_hist[0] = offset1;
         new_dict.offset_hist[1] = offset2;
         new_dict.offset_hist[2] = offset3;
@@ -102,3 +157,102 @@ impl Dictionary {
         Ok(new_dict)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn offset_history_start(raw: &[u8]) -> usize {
+        let mut huf = crate::decoding::scratch::HuffmanScratch::new();
+        let mut fse = crate::decoding::scratch::FSEScratch::new();
+        let mut cursor = 8usize;
+
+        let huf_size = huf
+            .table
+            .build_decoder(&raw[cursor..])
+            .expect("reference dictionary huffman table should decode");
+        cursor += huf_size as usize;
+
+        let of_size = fse
+            .offsets
+            .build_decoder(
+                &raw[cursor..],
+                crate::decoding::sequence_section_decoder::OF_MAX_LOG,
+            )
+            .expect("reference dictionary OF table should decode");
+        cursor += of_size;
+
+        let ml_size = fse
+            .match_lengths
+            .build_decoder(
+                &raw[cursor..],
+                crate::decoding::sequence_section_decoder::ML_MAX_LOG,
+            )
+            .expect("reference dictionary ML table should decode");
+        cursor += ml_size;
+
+        let ll_size = fse
+            .literal_lengths
+            .build_decoder(
+                &raw[cursor..],
+                crate::decoding::sequence_section_decoder::LL_MAX_LOG,
+            )
+            .expect("reference dictionary LL table should decode");
+        cursor += ll_size;
+
+        cursor
+    }
+
+    #[test]
+    fn decode_dict_rejects_short_buffer_before_magic_and_id() {
+        let err = match Dictionary::decode_dict(&[]) {
+            Ok(_) => panic!("expected short dictionary to fail"),
+            Err(err) => err,
+        };
+        assert!(matches!(
+            err,
+            DictionaryDecodeError::DictionaryTooSmall { got: 0, need: 8 }
+        ));
+    }
+
+    #[test]
+    fn decode_dict_malformed_input_returns_error_instead_of_panicking() {
+        let mut raw = Vec::new();
+        raw.extend_from_slice(&MAGIC_NUM);
+        raw.extend_from_slice(&1u32.to_le_bytes());
+        raw.extend_from_slice(&[0u8; 7]);
+
+        let result = std::panic::catch_unwind(|| Dictionary::decode_dict(&raw));
+        assert!(
+            result.is_ok(),
+            "decode_dict must not panic on malformed input"
+        );
+        assert!(
+            result.unwrap().is_err(),
+            "malformed dictionary must return error"
+        );
+    }
+
+    #[test]
+    fn decode_dict_rejects_zero_repeat_offsets() {
+        let mut raw = include_bytes!("../../dict_tests/dictionary").to_vec();
+        let offset_start = offset_history_start(&raw);
+
+        // Corrupt rep0 to zero.
+        raw[offset_start..offset_start + 4].copy_from_slice(&0u32.to_le_bytes());
+        let decoded = Dictionary::decode_dict(&raw);
+        assert!(matches!(
+            decoded,
+            Err(DictionaryDecodeError::ZeroRepeatOffsetInDictionary { index: 0 })
+        ));
+    }
+
+    #[test]
+    fn from_raw_content_rejects_empty_dictionary_content() {
+        let result = Dictionary::from_raw_content(1, Vec::new());
+        assert!(matches!(
+            result,
+            Err(DictionaryDecodeError::DictionaryTooSmall { got: 0, need: 1 })
+        ));
+    }
+}

zstd/src/decoding/errors.rs

@@ -425,6 +425,9 @@ impl core::fmt::Display for DecodeBufferError {
 #[non_exhaustive]
 pub enum DictionaryDecodeError {
     BadMagicNum { got: [u8; 4] },
+    DictionaryTooSmall { got: usize, need: usize },
+    ZeroDictionaryId,
+    ZeroRepeatOffsetInDictionary { index: u8 },
     FSETableError(FSETableError),
     HuffmanTableError(HuffmanTableError),
 }
@@ -451,6 +454,18 @@ impl core::fmt::Display for DictionaryDecodeError {
                     crate::decoding::dictionary::MAGIC_NUM,
                 )
             }
+            DictionaryDecodeError::DictionaryTooSmall { got, need } => {
+                write!(
+                    f,
+                    "Dictionary is too small: got {got} bytes, need at least {need} bytes",
+                )
+            }
+            DictionaryDecodeError::ZeroDictionaryId => {
+                write!(f, "Dictionary id must be non-zero")
+            }
+            DictionaryDecodeError::ZeroRepeatOffsetInDictionary { index } => {
+                write!(f, "Dictionary repeat offset rep{index} must be non-zero")
+            }
             DictionaryDecodeError::FSETableError(e) => write!(f, "{e:?}"),
             DictionaryDecodeError::HuffmanTableError(e) => write!(f, "{e:?}"),
         }