From 37b2b1dcc3a2d0338ab0a6d00fe4815a706cd90d Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 15 Oct 2024 10:28:38 +0800 Subject: [PATCH] [VL] Lower default overhead ratio and spill run size --- .../org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 2 +- cpp/velox/compute/WholeStageResultIterator.cc | 2 +- .../common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index e763e31dc5a9..68fc14b6e470 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -51,7 +51,7 @@ class VeloxListenerApi extends ListenerApi with Logging { // Overhead memory limits. val offHeapSize = conf.getSizeAsBytes(GlutenConfig.SPARK_OFFHEAP_SIZE_KEY) - val desiredOverheadSize = (0.3 * offHeapSize).toLong.max(ByteUnit.MiB.toBytes(384)) + val desiredOverheadSize = (0.1 * offHeapSize).toLong.max(ByteUnit.MiB.toBytes(384)) if (!SparkResourceUtil.isMemoryOverheadSet(conf)) { // If memory overhead is not set by user, automatically set it according to off-heap settings. logInfo( diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index ea5bd59794f1..0a7f19774bb3 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -491,7 +491,7 @@ std::unordered_map WholeStageResultIterator::getQueryC configs[velox::core::QueryConfig::kMaxSpillFileSize] = std::to_string(veloxCfg_->get(kMaxSpillFileSize, 1L * 1024 * 1024 * 1024)); configs[velox::core::QueryConfig::kMaxSpillRunRows] = - std::to_string(veloxCfg_->get(kMaxSpillRunRows, 3L * 1024 * 1024)); + std::to_string(veloxCfg_->get(kMaxSpillRunRows, 1L * 1024 * 1024)); configs[velox::core::QueryConfig::kMaxSpillBytes] = std::to_string(veloxCfg_->get(kMaxSpillBytes, 107374182400LL)); configs[velox::core::QueryConfig::kSpillWriteBufferSize] = diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index aa54f28507fb..ad6f6a2a2032 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -1499,7 +1499,7 @@ object GlutenConfig { .internal() .doc("The maximum row size of a single spill run") .bytesConf(ByteUnit.BYTE) - .createWithDefaultString("3M") + .createWithDefaultString("1M") val COLUMNAR_VELOX_MAX_SPILL_BYTES = buildConf("spark.gluten.sql.columnar.backend.velox.maxSpillBytes")