From e78a66c4525022fd29ec4a57891724e5d1098d26 Mon Sep 17 00:00:00 2001 From: Sureshkumar Karuppuchamy Date: Wed, 28 Jan 2026 21:55:21 -0800 Subject: [PATCH] fix: downgrade spill memory warning from warn to debug (#19846) Downgrade the log level for record batch memory usage warnings from warn! to debug! to reduce production log pollution. This warning was being triggered excessively in production when record batch memory usage exceeded expected limits, creating noise without affecting functional correctness. The diagnostic information remains available at debug level for development and troubleshooting purposes. Fixes #19846 Co-Authored-By: Claude (claude-sonnet-4.5) --- datafusion/physical-plan/src/spill/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs index 166805a337349..59938c3e8cd73 100644 --- a/datafusion/physical-plan/src/spill/mod.rs +++ b/datafusion/physical-plan/src/spill/mod.rs @@ -49,7 +49,7 @@ use datafusion_common_runtime::SpawnedTask; use datafusion_execution::RecordBatchStream; use datafusion_execution::disk_manager::RefCountedTempFile; use futures::{FutureExt as _, Stream}; -use log::warn; +use log::debug; /// Stream that reads spill files from disk where each batch is read in a spawned blocking task /// It will read one batch at a time and will not do any buffering, to buffer data use [`crate::common::spawn_buffered`] @@ -154,7 +154,7 @@ impl SpillReaderStream { > max_record_batch_memory + SPILL_BATCH_MEMORY_MARGIN { - warn!( + debug!( "Record batch memory usage ({actual_size} bytes) exceeds the expected limit ({max_record_batch_memory} bytes) \n\ by more than the allowed tolerance ({SPILL_BATCH_MEMORY_MARGIN} bytes).\n\ This likely indicates a bug in memory accounting during spilling.\n\