From a46cfde43ee38e45f1c9ad58c727998a9c2b93b6 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Sat, 18 Apr 2026 16:27:36 +0800 Subject: [PATCH 1/2] comment to `read_spill_as_stream` --- .../physical-plan/src/spill/spill_manager.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rs index c81c01bbe31a3..cae8b6fb0826e 100644 --- a/datafusion/physical-plan/src/spill/spill_manager.rs +++ b/datafusion/physical-plan/src/spill/spill_manager.rs @@ -160,9 +160,22 @@ impl SpillManager { Ok(file.map(|f| (f, max_record_batch_size))) } - /// Reads a spill file as a stream. The file must be created by the current `SpillManager`. - /// This method will generate output in FIFO order: the batch appended first - /// will be read first. + /// Reads a spill file as a stream. The file must be created by the current + /// `SpillManager`. + /// + /// Output is produced in FIFO order: the batch appended first is read first. + /// + /// # Arg `max_record_batch_memory` + /// + /// Most callers should pass `None`. This is mainly useful for the + /// memory-limited sort-preserving merge path. + /// + /// When provided, this value is used only as a validation hint. If a + /// decoded batch exceeds this threshold, a debug-level log message is + /// emitted. + /// + /// That path uses the maximum spilled batch size to conservatively estimate + /// the merge degree when merging multiple sorted runs. pub fn read_spill_as_stream( &self, spill_file_path: RefCountedTempFile, From 8a89c0c090fcda1d7925a21ab10471f641c89e8f Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Sun, 19 Apr 2026 10:16:38 +0800 Subject: [PATCH 2/2] review: more comment --- datafusion/physical-plan/src/spill/spill_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rs index cae8b6fb0826e..1664256e65888 100644 --- a/datafusion/physical-plan/src/spill/spill_manager.rs +++ b/datafusion/physical-plan/src/spill/spill_manager.rs @@ -161,7 +161,7 @@ impl SpillManager { } /// Reads a spill file as a stream. The file must be created by the current - /// `SpillManager`. + /// `SpillManager`; otherwise behavior is undefined. /// /// Output is produced in FIFO order: the batch appended first is read first. ///