From 793950141dccfc6a470a21074ebf61399cc151d2 Mon Sep 17 00:00:00 2001 From: Juliusz Sompolski Date: Tue, 18 Jul 2023 12:20:39 +0200 Subject: [PATCH 1/2] reuse method --- .../apache/spark/sql/connect/utils/MetricGenerator.scala | 6 ------ .../sql/execution/adaptive/AdaptiveSparkPlanHelper.scala | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala index 88120e616efdb..7f5e1ad9d361b 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala @@ -47,12 +47,6 @@ private[connect] object MetricGenerator extends AdaptiveSparkPlanHelper { allChildren(p).flatMap(c => transformPlan(c, p.id)) } - private def allChildren(p: SparkPlan): Seq[SparkPlan] = p match { - case a: AdaptiveSparkPlanExec => Seq(a.executedPlan) - case s: QueryStageExec => Seq(s.plan) - case _ => p.children - } - private def transformPlan( p: SparkPlan, parentId: Int): Seq[ExecutePlanResponse.Metrics.MetricObject] = { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala index eecfa40e8d0bd..c58d925f28e5c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala @@ -122,7 +122,7 @@ trait AdaptiveSparkPlanHelper { subqueries ++ subqueries.flatMap(subqueriesAll) } - private def allChildren(p: SparkPlan): Seq[SparkPlan] = p match { + protected def allChildren(p: SparkPlan): Seq[SparkPlan] = p match { case a: AdaptiveSparkPlanExec => Seq(a.executedPlan) case s: QueryStageExec => Seq(s.plan) case _ => p.children From 0aff9a3bf6a60a6daf124029de6d3650993879ab Mon Sep 17 00:00:00 2001 From: Juliusz Sompolski Date: Tue, 18 Jul 2023 13:48:47 +0200 Subject: [PATCH 2/2] unused imports --- .../org/apache/spark/sql/connect/utils/MetricGenerator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala index 7f5e1ad9d361b..6395fb588ab84 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/utils/MetricGenerator.scala @@ -22,7 +22,7 @@ import scala.collection.JavaConverters._ import org.apache.spark.connect.proto.ExecutePlanResponse import org.apache.spark.sql.DataFrame import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, QueryStageExec} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper /** * Helper object for generating responses with metrics from queries.