From 584dc717fe0873f43d7757d81d1059c8910d868f Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 5 Mar 2025 19:15:10 +0800 Subject: [PATCH 1/2] Initial --- .../sql/execution/GlutenAutoAdjustStageResourceProfile.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala index 1e9197b6101c..58c3750bbf87 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala @@ -88,6 +88,11 @@ case class GlutenAutoAdjustStageResourceProfile(glutenConf: GlutenConfig, spark: val newExecutorMemory = new ExecutorResourceRequest(ResourceProfile.MEMORY, newMemoryAmount.toLong) executorResource.put(ResourceProfile.MEMORY, newExecutorMemory) + + val newExecutorOffheap = + new ExecutorResourceRequest(ResourceProfile.OFFHEAP_MEM, offheapRequest.get.amount / 10) + executorResource.put(ResourceProfile.OFFHEAP_MEM, newExecutorOffheap) + val newRP = new ResourceProfile(executorResource.toMap, taskResource.toMap) return applyNewResourceProfileIfPossible(plan, newRP, rpManager) } From 3dde772953f22b5b5d4dc02eb6301e86ad979f8a Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 2 Apr 2025 13:40:49 +0800 Subject: [PATCH 2/2] Update doc --- .../execution/GlutenAutoAdjustStageResourceProfile.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala index 58c3750bbf87..c619266915bc 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenAutoAdjustStageResourceProfile.scala @@ -37,10 +37,10 @@ import scala.collection.mutable.ArrayBuffer /** * This rule is used to dynamic adjust stage resource profile for following purposes: - * 1. swap offheap and onheap memory size when whole stage fallback happened 2. increase executor - * heap memory if stage contains gluten operator and spark operator at the same time. Note: we - * don't support set resource profile for final stage now. Todo: support set resource profile - * for final stage. + * 1. Decrease offheap and increase onheap memory size when whole stage fallback happened; 2. + * Increase executor heap memory if stage contains gluten operator and spark operator at the + * same time. Note: we don't support set resource profile for final stage now. Todo: will + * support it. */ @Experimental case class GlutenAutoAdjustStageResourceProfile(glutenConf: GlutenConfig, spark: SparkSession)