perf(fse): pack decoder entries and align decode tables

zstd/src/decoding/scratch.rs

@@ -6,6 +6,7 @@ use crate::decoding::dictionary::Dictionary;
 use crate::fse::FSETable;
 use crate::huff0::HuffmanTable;
 use alloc::vec::Vec;
+use core::ops::{Deref, DerefMut};
 
 use crate::blocks::sequence_section::{
     MAX_LITERAL_LENGTH_CODE, MAX_MATCH_LENGTH_CODE, MAX_OFFSET_CODE,
@@ -33,11 +34,11 @@ impl DecoderScratch {
                 table: HuffmanTable::new(),
             },
             fse: FSEScratch {
-                offsets: FSETable::new(MAX_OFFSET_CODE),
+                offsets: AlignedFSETable::new(MAX_OFFSET_CODE),
                 of_rle: None,
-                literal_lengths: FSETable::new(MAX_LITERAL_LENGTH_CODE),
+                literal_lengths: AlignedFSETable::new(MAX_LITERAL_LENGTH_CODE),
                 ll_rle: None,
-                match_lengths: FSETable::new(MAX_MATCH_LENGTH_CODE),
+                match_lengths: AlignedFSETable::new(MAX_MATCH_LENGTH_CODE),
                 ml_rle: None,
             },
             buffer: DecodeBuffer::new(window_size),
@@ -97,22 +98,22 @@ impl Default for HuffmanScratch {
 }
 
 pub struct FSEScratch {
-    pub offsets: FSETable,
+    pub offsets: AlignedFSETable,
     pub of_rle: Option<u8>,
-    pub literal_lengths: FSETable,
+    pub literal_lengths: AlignedFSETable,
     pub ll_rle: Option<u8>,
-    pub match_lengths: FSETable,
+    pub match_lengths: AlignedFSETable,
     pub ml_rle: Option<u8>,
 }
 
 impl FSEScratch {
     pub fn new() -> FSEScratch {
         FSEScratch {
-            offsets: FSETable::new(MAX_OFFSET_CODE),
+            offsets: AlignedFSETable::new(MAX_OFFSET_CODE),
             of_rle: None,
-            literal_lengths: FSETable::new(MAX_LITERAL_LENGTH_CODE),
+            literal_lengths: AlignedFSETable::new(MAX_LITERAL_LENGTH_CODE),
             ll_rle: None,
-            match_lengths: FSETable::new(MAX_MATCH_LENGTH_CODE),
+            match_lengths: AlignedFSETable::new(MAX_MATCH_LENGTH_CODE),
             ml_rle: None,
         }
     }
@@ -132,3 +133,30 @@ impl Default for FSEScratch {
         Self::new()
     }
 }
+
+// Keep LL/ML/OF table *objects* cache-line aligned to avoid cross-table placement
+// effects in DecoderScratch when they are accessed in the same decode hot loop.
+// Note: this aligns the table containers, not the `Vec<Entry>` backing allocations.
+#[cfg_attr(target_arch = "aarch64", repr(align(128)))]
+#[cfg_attr(not(target_arch = "aarch64"), repr(align(64)))]
+pub struct AlignedFSETable(FSETable);
+
+impl AlignedFSETable {
+    fn new(max_symbol: u8) -> Self {
+        Self(FSETable::new(max_symbol))
+    }
+}
+
+impl Deref for AlignedFSETable {
+    type Target = FSETable;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for AlignedFSETable {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}

zstd/src/decoding/sequence_section_decoder.rs

@@ -475,7 +475,7 @@ fn test_ll_default() {
             idx,
             table.decode[idx].symbol,
             table.decode[idx].num_bits,
-            table.decode[idx].base_line
+            table.decode[idx].new_state
         );
     }
 
@@ -484,21 +484,21 @@ fn test_ll_default() {
     //just test a few values. TODO test all values
     assert!(table.decode[0].symbol == 0);
     assert!(table.decode[0].num_bits == 4);
-    assert!(table.decode[0].base_line == 0);
+    assert!(table.decode[0].new_state == 0);
 
     assert!(table.decode[19].symbol == 27);
     assert!(table.decode[19].num_bits == 6);
-    assert!(table.decode[19].base_line == 0);
+    assert!(table.decode[19].new_state == 0);
 
     assert!(table.decode[39].symbol == 25);
     assert!(table.decode[39].num_bits == 4);
-    assert!(table.decode[39].base_line == 16);
+    assert!(table.decode[39].new_state == 16);
 
     assert!(table.decode[60].symbol == 35);
     assert!(table.decode[60].num_bits == 6);
-    assert!(table.decode[60].base_line == 0);
+    assert!(table.decode[60].new_state == 0);
 
     assert!(table.decode[59].symbol == 24);
     assert!(table.decode[59].num_bits == 5);
-    assert!(table.decode[59].base_line == 32);
+    assert!(table.decode[59].new_state == 32);
 }