From 3df9f7d93e275f32abd33870257c9e1be96e35d8 Mon Sep 17 00:00:00 2001 From: hushengquan <1390305506@qq.com> Date: Tue, 14 Apr 2026 20:27:04 +0800 Subject: [PATCH] perf: submit I/O requests eagerly in FullZipScheduler --- .../src/encodings/logical/primitive.rs | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/rust/lance-encoding/src/encodings/logical/primitive.rs b/rust/lance-encoding/src/encodings/logical/primitive.rs index 6bcdbc5e56b..9f6be591d13 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive.rs @@ -2165,15 +2165,17 @@ impl FullZipScheduler { } fn create_page_load_task( - read_source: FullZipReadSource, - byte_ranges: Vec>, - priority: u64, + io_future: BoxFuture<'static, Result>>, num_rows: u64, details: Arc, bits_per_offset: u8, ) -> PageLoadTask { let load_task = async move { - let data = read_source.fetch(&byte_ranges, priority).await?; + let buffers = io_future.await?; + let data = buffers + .into_iter() + .map(|bytes| LanceBuffer::from_bytes(bytes, 1)) + .collect::>(); Self::create_decoder(details, data, num_rows, bits_per_offset) } .boxed(); @@ -2333,14 +2335,9 @@ impl FullZipScheduler { rep_index.bytes_per_value, data_buf_position, ); - let page_load_task = Self::create_page_load_task( - FullZipReadSource::Remote(io.clone()), - byte_ranges, - priority, - num_rows, - details, - bits_per_offset, - ); + let io_future = io.submit_request(byte_ranges, priority); + let page_load_task = + Self::create_page_load_task(io_future, num_rows, details, bits_per_offset); return Ok(vec![page_load_task]); } @@ -2403,10 +2400,9 @@ impl FullZipScheduler { }) .collect::>(); + let io_future = io.submit_request(byte_ranges, self.priority); let page_load_task = Self::create_page_load_task( - FullZipReadSource::Remote(io.clone()), - byte_ranges, - self.priority, + io_future, num_rows, self.details.clone(), self.bits_per_offset,