From 0a1871e2ee35964bea6c9783fd3f77c86dce4d11 Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Sat, 27 Dec 2025 15:10:23 +0530 Subject: [PATCH 01/12] Allow Spark partial / Comet final for compatible aggregates --- .../serde/CometAggregateExpressionSerde.scala | 14 ++++++++ .../apache/comet/serde/QueryPlanSerde.scala | 17 ++++++++++ .../org/apache/comet/serde/aggregates.scala | 25 ++++++++++++++ .../apache/spark/sql/comet/operators.scala | 11 +++--- .../comet/rules/CometExecRuleSuite.scala | 34 +++++++++++++++++-- 5 files changed, 95 insertions(+), 6 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala b/spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala index 0a5a2770b4..fc8f776ca7 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala @@ -49,6 +49,20 @@ trait CometAggregateExpressionSerde[T <: AggregateFunction] { */ def getSupportLevel(expr: T): SupportLevel = Compatible(None) + /** + * Indicates whether this aggregate function supports "Spark partial / Comet final" mixed + * execution. This requires the intermediate buffer format to be compatible between Spark and + * Comet. + * + * Only aggregates with simple, compatible intermediate buffers should return true. Aggregates + * with complex buffers or those with known incompatibilities (e.g., decimal overflow handling + * differences) should return false. + * + * @return + * true if the aggregate can safely run with Spark partial and Comet final, false otherwise + */ + def supportsSparkPartialCometFinal: Boolean = false + /** * Convert a Spark expression into a protocol buffer representation that can be passed into * native code. diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 59fb0f9819..c00f41021c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -271,6 +271,23 @@ object QueryPlanSerde extends Logging with CometExprShim { classOf[VariancePop] -> CometVariancePop, classOf[VarianceSamp] -> CometVarianceSamp) + /** + * Checks if the given aggregate function supports "Spark partial / Comet final" mixed + * execution. This is used to determine if Comet can process a final aggregate even when the + * partial aggregate was performed by Spark. + * + * @param fn + * The aggregate function to check + * @return + * true if the aggregate supports mixed execution, false otherwise + */ + def aggSupportsMixedExecution(fn: AggregateFunction): Boolean = { + aggrSerdeMap.get(fn.getClass) match { + case Some(handler) => handler.supportsSparkPartialCometFinal + case None => false + } + } + // A unique id for each expression. ~used to look up QueryContext during error creation. private val exprIdCounter = new AtomicLong(0) diff --git a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala index 1485589b46..4f9d5a3c8a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala @@ -34,6 +34,9 @@ import org.apache.comet.shims.CometEvalModeUtil object CometMin extends CometAggregateExpressionSerde[Min] { + // Min has a simple intermediate buffer (single value) compatible between Spark and Comet + override def supportsSparkPartialCometFinal: Boolean = true + override def convert( aggExpr: AggregateExpression, expr: Min, @@ -81,6 +84,9 @@ object CometMin extends CometAggregateExpressionSerde[Min] { object CometMax extends CometAggregateExpressionSerde[Max] { + // Max has a simple intermediate buffer (single value) compatible between Spark and Comet + override def supportsSparkPartialCometFinal: Boolean = true + override def convert( aggExpr: AggregateExpression, expr: Max, @@ -127,6 +133,10 @@ object CometMax extends CometAggregateExpressionSerde[Max] { } object CometCount extends CometAggregateExpressionSerde[Count] { + + // Count has a simple intermediate buffer (single Long) compatible between Spark and Comet + override def supportsSparkPartialCometFinal: Boolean = true + override def convert( aggExpr: AggregateExpression, expr: Count, @@ -306,6 +316,11 @@ object CometLast extends CometAggregateExpressionSerde[Last] { } object CometBitAndAgg extends CometAggregateExpressionSerde[BitAndAgg] { + + // BitAnd has a simple intermediate buffer (single integral value) + // compatible between Spark and Comet + override def supportsSparkPartialCometFinal: Boolean = true + override def convert( aggExpr: AggregateExpression, bitAnd: BitAndAgg, @@ -340,6 +355,11 @@ object CometBitAndAgg extends CometAggregateExpressionSerde[BitAndAgg] { } object CometBitOrAgg extends CometAggregateExpressionSerde[BitOrAgg] { + + // BitOr has a simple intermediate buffer (single integral value) + // compatible between Spark and Comet + override def supportsSparkPartialCometFinal: Boolean = true + override def convert( aggExpr: AggregateExpression, bitOr: BitOrAgg, @@ -374,6 +394,11 @@ object CometBitOrAgg extends CometAggregateExpressionSerde[BitOrAgg] { } object CometBitXOrAgg extends CometAggregateExpressionSerde[BitXorAgg] { + + // BitXor has a simple intermediate buffer (single integral value) + // compatible between Spark and Comet + override def supportsSparkPartialCometFinal: Boolean = true + override def convert( aggExpr: AggregateExpression, bitXor: BitXorAgg, diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index 1f5e7b6677..48a226ded4 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -55,7 +55,7 @@ import org.apache.comet.CometSparkSessionExtensions.{isCometShuffleEnabled, with import org.apache.comet.parquet.CometParquetUtils import org.apache.comet.serde.{CometOperatorSerde, Compatible, Incompatible, OperatorOuterClass, SupportLevel, Unsupported} import org.apache.comet.serde.OperatorOuterClass.{AggregateMode => CometAggregateMode, Operator} -import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, exprToProto, supportedSortType} +import org.apache.comet.serde.QueryPlanSerde.{aggExprToProto, aggSupportsMixedExecution, exprToProto, supportedSortType} import org.apache.comet.serde.operator.CometSink /** @@ -1343,11 +1343,14 @@ trait CometBaseAggregate { val modes = aggregate.aggregateExpressions.map(_.mode).distinct // In distinct aggregates there can be a combination of modes val multiMode = modes.size > 1 - // For a final mode HashAggregate, we only need to transform the HashAggregate - // if there is Comet partial aggregation. + // For a final mode HashAggregate, check if there is Comet partial aggregation. + // If not, we can still proceed if all aggregates support mixed execution + // (Spark partial / Comet final). See https://github.com/apache/datafusion-comet/issues/2894 val sparkFinalMode = modes.contains(Final) && findCometPartialAgg(aggregate.child).isEmpty + val allSupportMixedExecution = aggregate.aggregateExpressions.forall(expr => + aggSupportsMixedExecution(expr.aggregateFunction)) - if (multiMode || sparkFinalMode) { + if (multiMode || (sparkFinalMode && !allSupportMixedExecution)) { return None } diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index cf6f8918f4..a343f7fb93 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -157,7 +157,9 @@ class CometExecRuleSuite extends CometTestBase { } } - test("CometExecRule should not allow Spark partial and Comet final hash aggregate") { + test("CometExecRule should not allow Spark partial and Comet final for unsafe aggregates") { + // https://github.com/apache/datafusion-comet/issues/2894 + // SUM is not safe for mixed execution due to potential overflow handling differences withTempView("test_data") { createTestDataFrame.createOrReplaceTempView("test_data") @@ -173,7 +175,7 @@ class CometExecRuleSuite extends CometTestBase { CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") { val transformedPlan = applyCometExecRule(sparkPlan) - // if the partial aggregate cannot be converted to Comet, then neither should be + // SUM is not safe for mixed execution, so both partial and final should fall back assert( countOperators(transformedPlan, classOf[HashAggregateExec]) == originalHashAggCount) assert(countOperators(transformedPlan, classOf[CometHashAggregateExec]) == 0) @@ -181,6 +183,34 @@ class CometExecRuleSuite extends CometTestBase { } } + test("CometExecRule should allow Spark partial and Comet final for safe aggregates") { + // https://github.com/apache/datafusion-comet/issues/2894 + // MIN, MAX, COUNT are safe for mixed execution (simple intermediate buffer) + withTempView("test_data") { + createTestDataFrame.createOrReplaceTempView("test_data") + + val sparkPlan = + createSparkPlan( + spark, + "SELECT MIN(id), MAX(id), COUNT(*) FROM test_data GROUP BY (id % 3)") + + // Count original Spark operators (should be 2: partial + final) + val originalHashAggCount = countOperators(sparkPlan, classOf[HashAggregateExec]) + assert(originalHashAggCount == 2) + + withSQLConf( + CometConf.COMET_ENABLE_PARTIAL_HASH_AGGREGATE.key -> "false", + CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") { + val transformedPlan = applyCometExecRule(sparkPlan) + + // MIN, MAX, COUNT support mixed execution, so final should run in Comet + // Partial stays in Spark (1 HashAggregateExec), final runs in Comet (1 CometHashAggregateExec) + assert(countOperators(transformedPlan, classOf[HashAggregateExec]) == 1) + assert(countOperators(transformedPlan, classOf[CometHashAggregateExec]) == 1) + } + } + } + test("CometExecRule should apply broadcast exchange transformations") { withTempView("test_data") { createTestDataFrame.createOrReplaceTempView("test_data") From d33a1a4f9448f49b36394cebf284de45db5eb7d4 Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Fri, 9 Jan 2026 18:47:16 +0530 Subject: [PATCH 02/12] Add unit tests for aggSupportsMixedExecution including bitwise aggregates --- .../comet/rules/CometExecRuleSuite.scala | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index a343f7fb93..4c3980165c 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -22,6 +22,8 @@ package org.apache.comet.rules import scala.util.Random import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal} +import org.apache.spark.sql.catalyst.expressions.aggregate.{Average, BitAndAgg, BitOrAgg, BitXorAgg, Count, Max, Min, Sum} import org.apache.spark.sql.comet._ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec import org.apache.spark.sql.execution._ @@ -29,8 +31,10 @@ import org.apache.spark.sql.execution.adaptive.QueryStageExec import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.types.{DataTypes, StructField, StructType} +import org.apache.spark.sql.types.IntegerType import org.apache.comet.CometConf +import org.apache.comet.serde.QueryPlanSerde import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** @@ -211,6 +215,65 @@ class CometExecRuleSuite extends CometTestBase { } } + test("QueryPlanSerde.aggSupportsMixedExecution should return true for safe aggregates") { + // Test aggregates that support Spark partial / Comet final execution + val testAttr = AttributeReference("id", IntegerType, nullable = true)() + assert( + QueryPlanSerde.aggSupportsMixedExecution(Min(testAttr)), + "Min should support mixed execution") + assert( + QueryPlanSerde.aggSupportsMixedExecution(Max(testAttr)), + "Max should support mixed execution") + assert( + QueryPlanSerde.aggSupportsMixedExecution(Count(testAttr)), + "Count should support mixed execution") + assert( + QueryPlanSerde.aggSupportsMixedExecution(BitAndAgg(testAttr)), + "BitAndAgg should support mixed execution") + assert( + QueryPlanSerde.aggSupportsMixedExecution(BitOrAgg(testAttr)), + "BitOrAgg should support mixed execution") + assert( + QueryPlanSerde.aggSupportsMixedExecution(BitXorAgg(testAttr)), + "BitXorAgg should support mixed execution") + } + + test("QueryPlanSerde.aggSupportsMixedExecution should return true for bitwise aggregates") { + // Test bitwise aggregates that support Spark partial / Comet final execution + val testAttr = AttributeReference("id", IntegerType, nullable = true)() + assert( + QueryPlanSerde.aggSupportsMixedExecution(BitAndAgg(testAttr)), + "BitAndAgg should support mixed execution") + assert( + QueryPlanSerde.aggSupportsMixedExecution(BitOrAgg(testAttr)), + "BitOrAgg should support mixed execution") + assert( + QueryPlanSerde.aggSupportsMixedExecution(BitXorAgg(testAttr)), + "BitXorAgg should support mixed execution") + } + + test("QueryPlanSerde.aggSupportsMixedExecution should return false for unsafe aggregates") { + // Test aggregates that don't support Spark partial / Comet final execution + val testAttr = AttributeReference("id", IntegerType, nullable = true)() + assert( + !QueryPlanSerde.aggSupportsMixedExecution(Sum(testAttr)), + "Sum should not support mixed execution") + assert( + !QueryPlanSerde.aggSupportsMixedExecution(Average(testAttr)), + "Average should not support mixed execution") + } + + test( + "QueryPlanSerde.aggSupportsMixedExecution should return false for aggregates with default implementation") { + // Test with an aggregate function that's in the map but doesn't override supportsSparkPartialCometFinal + val testAttr = AttributeReference("id", IntegerType, nullable = true)() + val firstAgg = new org.apache.spark.sql.catalyst.expressions.aggregate.First(testAttr, true) + // First is in the map but doesn't override supportsSparkPartialCometFinal, so should return false + assert( + !QueryPlanSerde.aggSupportsMixedExecution(firstAgg), + "First should not support mixed execution (default false)") + } + test("CometExecRule should apply broadcast exchange transformations") { withTempView("test_data") { createTestDataFrame.createOrReplaceTempView("test_data") From d88a71b9dec6938e0afe4aad603b9580b6746946 Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Thu, 22 Jan 2026 21:17:55 +0530 Subject: [PATCH 03/12] Fix build errors: Remove unused imports and fix feature flag conflicts --- .../test/scala/org/apache/comet/rules/CometExecRuleSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index 4c3980165c..1c6283a5fe 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -22,7 +22,7 @@ package org.apache.comet.rules import scala.util.Random import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal} +import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.catalyst.expressions.aggregate.{Average, BitAndAgg, BitOrAgg, BitXorAgg, Count, Max, Min, Sum} import org.apache.spark.sql.comet._ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec From 5db1fed8fafd20b1f5c420ecd0121c2322d8d32e Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Tue, 17 Mar 2026 00:38:24 +0530 Subject: [PATCH 04/12] Add bitwise aggregate mixed execution test --- .../comet/exec/CometAggregateSuite.scala | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index be60f4aaee..186f396e01 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -1292,6 +1292,34 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("bitwise aggregates allow Spark partial and Comet final") { + withSQLConf( + CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_MODE.key -> "jvm", + CometConf.COMET_ENABLE_PARTIAL_HASH_AGGREGATE.key -> "false", + CometConf.COMET_ENABLE_FINAL_HASH_AGGREGATE.key -> "true", + CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") { + val table = "bitwise_mixed" + withTable(table) { + sql( + s"create table $table(col1 long, col2 int, col3 short, col4 byte, grp int) using parquet") + sql( + s"insert into $table values" + + "(4, 1, 1, 3, 0), (4, 1, 1, 3, 0), (3, 3, 1, 4, 1)," + + " (2, 4, 2, 5, 1), (1, 3, 2, 6, 0)") + + // Partial aggregate stays in Spark, final aggregate runs in Comet. + checkSparkAnswerAndNumOfAggregates( + s"SELECT grp, BIT_AND(col1), BIT_OR(col1), BIT_XOR(col1)," + + " BIT_AND(col2), BIT_OR(col2), BIT_XOR(col2)," + + " BIT_AND(col3), BIT_OR(col3), BIT_XOR(col3)," + + " BIT_AND(col4), BIT_OR(col4), BIT_XOR(col4)" + + s" FROM $table GROUP BY grp", + 1) + } + } + } + def setupAndTestAggregates( table: String, data: Seq[(Any, Any, Any)], From 5acbddef01559b8dfdb9081b70fa6cfab52e1cc8 Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Wed, 18 Mar 2026 10:59:10 +0530 Subject: [PATCH 05/12] minor change --- .../test/scala/org/apache/comet/exec/CometAggregateSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index 186f396e01..cd295477bf 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -1310,7 +1310,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { // Partial aggregate stays in Spark, final aggregate runs in Comet. checkSparkAnswerAndNumOfAggregates( - s"SELECT grp, BIT_AND(col1), BIT_OR(col1), BIT_XOR(col1)," + + "SELECT grp, BIT_AND(col1), BIT_OR(col1), BIT_XOR(col1)," + " BIT_AND(col2), BIT_OR(col2), BIT_XOR(col2)," + " BIT_AND(col3), BIT_OR(col3), BIT_XOR(col3)," + " BIT_AND(col4), BIT_OR(col4), BIT_XOR(col4)" + From 676ec350195b29f4842b42ff954d86bb18b4396d Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Sat, 28 Mar 2026 11:50:08 +0530 Subject: [PATCH 06/12] chore: update Spark 3.5 golden files for mixed aggregate execution --- .../q10.native_datafusion/extended.txt | 8 +- .../q10.native_iceberg_compat/extended.txt | 8 +- .../q23a.native_datafusion/extended.txt | 22 ++--- .../q23b.native_datafusion/extended.txt | 22 ++--- .../q34.native_datafusion/extended.txt | 86 +++++++++---------- .../q54.native_datafusion/extended.txt | 8 +- .../q6.native_datafusion/extended.txt | 10 +-- .../q69.native_datafusion/extended.txt | 8 +- .../q69.native_iceberg_compat/extended.txt | 8 +- .../q73.native_datafusion/extended.txt | 86 +++++++++---------- .../q87.native_datafusion/extended.txt | 6 +- .../q87.native_iceberg_compat/extended.txt | 6 +- .../q10a.native_datafusion/extended.txt | 8 +- .../q34.native_datafusion/extended.txt | 86 +++++++++---------- .../q6.native_datafusion/extended.txt | 10 +-- 15 files changed, 188 insertions(+), 194 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/extended.txt index 40a8ef4a6e..41dd87136e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -64,4 +64,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 54 eligible operators (38%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 54 eligible operators (42%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/extended.txt index c5125b1861..4f0144bb58 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -60,4 +60,4 @@ TakeOrderedAndProject +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 37 out of 54 eligible operators (68%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/extended.txt index 7040e78da1..1ff1e095fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/extended.txt @@ -20,10 +20,10 @@ CometNativeColumnarToRow : : : : +- CometFilter : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange - : : : +- Project - : : : +- Filter - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- Project @@ -52,8 +52,8 @@ CometNativeColumnarToRow : : +- CometProject : : +- CometFilter : : : +- Subquery - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- HashAggregate @@ -109,10 +109,10 @@ CometNativeColumnarToRow : : : +- Scan parquet spark_catalog.default.web_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] : : : +- ReusedSubquery : : +- BroadcastExchange - : : +- Project - : : +- Filter - : : +- HashAggregate - : : +- CometNativeColumnarToRow + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometHashAggregate : : +- CometColumnarExchange : : +- HashAggregate : : +- Project @@ -157,4 +157,4 @@ CometNativeColumnarToRow +- CometFilter +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 83 out of 138 eligible operators (60%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 90 out of 138 eligible operators (65%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/extended.txt index 188775e7df..4afebfdfb6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/extended.txt @@ -23,10 +23,10 @@ CometNativeColumnarToRow : : : : : +- CometFilter : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- BroadcastExchange - : : : : +- Project - : : : : +- Filter - : : : : +- HashAggregate - : : : : +- CometNativeColumnarToRow + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate : : : : +- Project @@ -55,8 +55,8 @@ CometNativeColumnarToRow : : : +- CometProject : : : +- CometFilter : : : : +- Subquery - : : : : +- HashAggregate - : : : : +- CometNativeColumnarToRow + : : : : +- CometNativeColumnarToRow + : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate : : : : +- HashAggregate @@ -139,10 +139,10 @@ CometNativeColumnarToRow : : : : +- Scan parquet spark_catalog.default.web_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] : : : : +- ReusedSubquery : : : +- BroadcastExchange - : : : +- Project - : : : +- Filter - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- Project @@ -209,4 +209,4 @@ CometNativeColumnarToRow +- CometFilter +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 131 out of 190 eligible operators (68%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 138 out of 190 eligible operators (72%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/extended.txt index 273efea475..0793cfc12f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- HashAggregate @@ -113,4 +113,4 @@ TakeOrderedAndProject : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 51 out of 96 eligible operators (53%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 53 out of 96 eligible operators (55%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/extended.txt index f2cdb50e03..d2f4f3dd01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/extended.txt @@ -1,7 +1,7 @@ -TakeOrderedAndProject -+- Filter - +- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -65,4 +65,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.item -Comet accelerated 39 out of 58 eligible operators (67%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 42 out of 58 eligible operators (72%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/extended.txt index f48c67d6e1..0982045255 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -63,4 +63,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 53 eligible operators (39%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 53 eligible operators (43%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/extended.txt index bf624b5ce3..0f83198ca3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -59,4 +59,4 @@ TakeOrderedAndProject +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Comet accelerated 35 out of 53 eligible operators (66%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/extended.txt index 7a9e62d57c..77390c4d24 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/extended.txt @@ -1,5 +1,5 @@ -HashAggregate -+- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -79,4 +79,4 @@ HashAggregate +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 28 out of 66 eligible operators (42%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 29 out of 66 eligible operators (43%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/extended.txt index 5fddd74768..92c8617461 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/extended.txt @@ -1,5 +1,5 @@ -HashAggregate -+- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -70,4 +70,4 @@ HashAggregate +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Comet accelerated 55 out of 66 eligible operators (83%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 56 out of 66 eligible operators (84%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/extended.txt index 9091536e96..45768ebf9a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -62,4 +62,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 52 eligible operators (40%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 52 eligible operators (44%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/extended.txt index f2cdb50e03..d2f4f3dd01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/extended.txt @@ -1,7 +1,7 @@ -TakeOrderedAndProject -+- Filter - +- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -65,4 +65,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.item -Comet accelerated 39 out of 58 eligible operators (67%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 42 out of 58 eligible operators (72%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file From 53aea3d6c1d20d8fe20b670ef32bd510faf1151a Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Sat, 28 Mar 2026 12:18:34 +0530 Subject: [PATCH 07/12] chore: update Spark 4.0 goldens --- .../q10.native_datafusion/extended.txt | 8 +- .../q10.native_iceberg_compat/extended.txt | 8 +- .../q23a.native_datafusion/extended.txt | 22 ++--- .../q23b.native_datafusion/extended.txt | 22 ++--- .../q34.native_datafusion/extended.txt | 86 +++++++++---------- .../q54.native_datafusion/extended.txt | 8 +- .../q6.native_datafusion/extended.txt | 10 +-- .../q69.native_datafusion/extended.txt | 8 +- .../q69.native_iceberg_compat/extended.txt | 8 +- .../q73.native_datafusion/extended.txt | 86 +++++++++---------- .../q87.native_datafusion/extended.txt | 6 +- .../q87.native_iceberg_compat/extended.txt | 6 +- .../q10a.native_datafusion/extended.txt | 8 +- .../q34.native_datafusion/extended.txt | 86 +++++++++---------- .../q6.native_datafusion/extended.txt | 10 +-- 15 files changed, 188 insertions(+), 194 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/extended.txt index 40a8ef4a6e..41dd87136e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -64,4 +64,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 54 eligible operators (38%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 54 eligible operators (42%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/extended.txt index c5125b1861..4f0144bb58 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -60,4 +60,4 @@ TakeOrderedAndProject +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 37 out of 54 eligible operators (68%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/extended.txt index 7040e78da1..1ff1e095fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/extended.txt @@ -20,10 +20,10 @@ CometNativeColumnarToRow : : : : +- CometFilter : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange - : : : +- Project - : : : +- Filter - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- Project @@ -52,8 +52,8 @@ CometNativeColumnarToRow : : +- CometProject : : +- CometFilter : : : +- Subquery - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- HashAggregate @@ -109,10 +109,10 @@ CometNativeColumnarToRow : : : +- Scan parquet spark_catalog.default.web_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] : : : +- ReusedSubquery : : +- BroadcastExchange - : : +- Project - : : +- Filter - : : +- HashAggregate - : : +- CometNativeColumnarToRow + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometHashAggregate : : +- CometColumnarExchange : : +- HashAggregate : : +- Project @@ -157,4 +157,4 @@ CometNativeColumnarToRow +- CometFilter +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 83 out of 138 eligible operators (60%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 90 out of 138 eligible operators (65%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/extended.txt index 188775e7df..4afebfdfb6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/extended.txt @@ -23,10 +23,10 @@ CometNativeColumnarToRow : : : : : +- CometFilter : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- BroadcastExchange - : : : : +- Project - : : : : +- Filter - : : : : +- HashAggregate - : : : : +- CometNativeColumnarToRow + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate : : : : +- Project @@ -55,8 +55,8 @@ CometNativeColumnarToRow : : : +- CometProject : : : +- CometFilter : : : : +- Subquery - : : : : +- HashAggregate - : : : : +- CometNativeColumnarToRow + : : : : +- CometNativeColumnarToRow + : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate : : : : +- HashAggregate @@ -139,10 +139,10 @@ CometNativeColumnarToRow : : : : +- Scan parquet spark_catalog.default.web_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] : : : : +- ReusedSubquery : : : +- BroadcastExchange - : : : +- Project - : : : +- Filter - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- Project @@ -209,4 +209,4 @@ CometNativeColumnarToRow +- CometFilter +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 131 out of 190 eligible operators (68%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 138 out of 190 eligible operators (72%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_datafusion/extended.txt index 8da62eb377..86b0b8ed0d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- HashAggregate @@ -117,4 +117,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 51 out of 100 eligible operators (51%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 53 out of 100 eligible operators (53%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_datafusion/extended.txt index 44f11f06fa..7f796ad8d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_datafusion/extended.txt @@ -1,7 +1,7 @@ -TakeOrderedAndProject -+- Filter - +- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -67,4 +67,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.item -Comet accelerated 39 out of 60 eligible operators (65%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 42 out of 60 eligible operators (70%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/extended.txt index f48c67d6e1..0982045255 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -63,4 +63,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 53 eligible operators (39%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 53 eligible operators (43%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/extended.txt index bf624b5ce3..0f83198ca3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -59,4 +59,4 @@ TakeOrderedAndProject +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Comet accelerated 35 out of 53 eligible operators (66%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/extended.txt index 7a9e62d57c..77390c4d24 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/extended.txt @@ -1,5 +1,5 @@ -HashAggregate -+- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -79,4 +79,4 @@ HashAggregate +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 28 out of 66 eligible operators (42%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 29 out of 66 eligible operators (43%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/extended.txt index 5fddd74768..92c8617461 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/extended.txt @@ -1,5 +1,5 @@ -HashAggregate -+- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -70,4 +70,4 @@ HashAggregate +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Comet accelerated 55 out of 66 eligible operators (83%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 56 out of 66 eligible operators (84%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/extended.txt index 9091536e96..45768ebf9a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -62,4 +62,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 52 eligible operators (40%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 52 eligible operators (44%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_datafusion/extended.txt index 44f11f06fa..7f796ad8d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_datafusion/extended.txt @@ -1,7 +1,7 @@ -TakeOrderedAndProject -+- Filter - +- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -67,4 +67,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.item -Comet accelerated 39 out of 60 eligible operators (65%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 42 out of 60 eligible operators (70%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file From 6f8e4e1bc3867224a314ab84e028faf4c05b5285 Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Sun, 29 Mar 2026 15:31:43 +0530 Subject: [PATCH 08/12] fix: reject grouping on nested types containing maps (ARRAY(MAP(...)), STRUCT containing MAP) --- .../apache/spark/sql/comet/operators.scala | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index 48a226ded4..562e793152 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -42,7 +42,7 @@ import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, HashJoin, ShuffledHashJoinExec, SortMergeJoinExec} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{ArrayType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, TimestampNTZType} +import org.apache.spark.sql.types.{ArrayType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, ShortType, StringType, StructType, TimestampNTZType} import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.SerializableConfiguration import org.apache.spark.util.io.ChunkedByteBuffer @@ -1365,11 +1365,21 @@ trait CometBaseAggregate { return None } - if (groupingExpressions.exists(expr => - expr.dataType match { - case _: MapType => true - case _ => false - })) { + // Aggregate expressions with filter are not supported yet. + if (aggregateExpressions.exists(_.filter.isDefined)) { + withInfo(aggregate, "Aggregate expression with filter is not supported") + return None + } + + def containsMapType(dt: DataType): Boolean = dt match { + case _: MapType => true + case a: ArrayType => containsMapType(a.elementType) + case s: StructType => s.fields.exists(f => containsMapType(f.dataType)) + case _ => false + } + val mapTypeGroupings = + groupingExpressions.filter(expr => containsMapType(expr.dataType)) + if (mapTypeGroupings.nonEmpty) { withInfo(aggregate, "Grouping on map types is not supported") return None } From 11500f148cfa0e7631f5834c025ebaeaa8dcd1db Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Mon, 30 Mar 2026 21:40:44 +0530 Subject: [PATCH 09/12] Regenerate Spark 3.4 golden files for plan stability tests --- .../q10.native_datafusion/extended.txt | 8 +- .../q10.native_iceberg_compat/extended.txt | 8 +- .../q23a.native_datafusion/extended.txt | 22 ++--- .../q23b.native_datafusion/extended.txt | 22 ++--- .../q34.native_datafusion/extended.txt | 86 +++++++++---------- .../q54.native_datafusion/extended.txt | 8 +- .../q6.native_datafusion/extended.txt | 10 +-- .../q69.native_datafusion/extended.txt | 8 +- .../q69.native_iceberg_compat/extended.txt | 8 +- .../q73.native_datafusion/extended.txt | 86 +++++++++---------- .../q87.native_datafusion/extended.txt | 6 +- .../q87.native_iceberg_compat/extended.txt | 6 +- .../q10a.native_datafusion/extended.txt | 8 +- .../q34.native_datafusion/extended.txt | 86 +++++++++---------- .../q6.native_datafusion/extended.txt | 10 +-- 15 files changed, 188 insertions(+), 194 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/extended.txt index 40a8ef4a6e..41dd87136e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -64,4 +64,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 54 eligible operators (38%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 54 eligible operators (42%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/extended.txt index c5125b1861..4f0144bb58 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -60,4 +60,4 @@ TakeOrderedAndProject +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Comet accelerated 35 out of 54 eligible operators (64%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 37 out of 54 eligible operators (68%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/extended.txt index 7040e78da1..1ff1e095fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/extended.txt @@ -20,10 +20,10 @@ CometNativeColumnarToRow : : : : +- CometFilter : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : +- BroadcastExchange - : : : +- Project - : : : +- Filter - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- Project @@ -52,8 +52,8 @@ CometNativeColumnarToRow : : +- CometProject : : +- CometFilter : : : +- Subquery - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- HashAggregate @@ -109,10 +109,10 @@ CometNativeColumnarToRow : : : +- Scan parquet spark_catalog.default.web_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] : : : +- ReusedSubquery : : +- BroadcastExchange - : : +- Project - : : +- Filter - : : +- HashAggregate - : : +- CometNativeColumnarToRow + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometHashAggregate : : +- CometColumnarExchange : : +- HashAggregate : : +- Project @@ -157,4 +157,4 @@ CometNativeColumnarToRow +- CometFilter +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 83 out of 138 eligible operators (60%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 90 out of 138 eligible operators (65%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/extended.txt index 188775e7df..4afebfdfb6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/extended.txt @@ -23,10 +23,10 @@ CometNativeColumnarToRow : : : : : +- CometFilter : : : : : +- CometNativeScan parquet spark_catalog.default.date_dim : : : : +- BroadcastExchange - : : : : +- Project - : : : : +- Filter - : : : : +- HashAggregate - : : : : +- CometNativeColumnarToRow + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate : : : : +- Project @@ -55,8 +55,8 @@ CometNativeColumnarToRow : : : +- CometProject : : : +- CometFilter : : : : +- Subquery - : : : : +- HashAggregate - : : : : +- CometNativeColumnarToRow + : : : : +- CometNativeColumnarToRow + : : : : +- CometHashAggregate : : : : +- CometColumnarExchange : : : : +- HashAggregate : : : : +- HashAggregate @@ -139,10 +139,10 @@ CometNativeColumnarToRow : : : : +- Scan parquet spark_catalog.default.web_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] : : : : +- ReusedSubquery : : : +- BroadcastExchange - : : : +- Project - : : : +- Filter - : : : +- HashAggregate - : : : +- CometNativeColumnarToRow + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometHashAggregate : : : +- CometColumnarExchange : : : +- HashAggregate : : : +- Project @@ -209,4 +209,4 @@ CometNativeColumnarToRow +- CometFilter +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 131 out of 190 eligible operators (68%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 138 out of 190 eligible operators (72%). Final plan contains 20 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/extended.txt index 273efea475..0793cfc12f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- HashAggregate @@ -113,4 +113,4 @@ TakeOrderedAndProject : +- CometNativeScan parquet spark_catalog.default.date_dim +- CometNativeScan parquet spark_catalog.default.date_dim -Comet accelerated 51 out of 96 eligible operators (53%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 53 out of 96 eligible operators (55%). Final plan contains 18 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/extended.txt index f2cdb50e03..d2f4f3dd01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/extended.txt @@ -1,7 +1,7 @@ -TakeOrderedAndProject -+- Filter - +- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -65,4 +65,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.item -Comet accelerated 39 out of 58 eligible operators (67%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 42 out of 58 eligible operators (72%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/extended.txt index f48c67d6e1..0982045255 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -63,4 +63,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 53 eligible operators (39%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 53 eligible operators (43%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/extended.txt index bf624b5ce3..0f83198ca3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -59,4 +59,4 @@ TakeOrderedAndProject +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Comet accelerated 35 out of 53 eligible operators (66%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 37 out of 53 eligible operators (69%). Final plan contains 7 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/extended.txt index 7a9e62d57c..77390c4d24 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/extended.txt @@ -1,5 +1,5 @@ -HashAggregate -+- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -79,4 +79,4 @@ HashAggregate +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 28 out of 66 eligible operators (42%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 29 out of 66 eligible operators (43%). Final plan contains 14 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/extended.txt index 5fddd74768..92c8617461 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/extended.txt @@ -1,5 +1,5 @@ -HashAggregate -+- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -70,4 +70,4 @@ HashAggregate +- CometFilter +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Comet accelerated 55 out of 66 eligible operators (83%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 56 out of 66 eligible operators (84%). Final plan contains 5 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/extended.txt index 9091536e96..45768ebf9a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/extended.txt @@ -1,6 +1,6 @@ -TakeOrderedAndProject -+- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -62,4 +62,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.customer_demographics -Comet accelerated 21 out of 52 eligible operators (40%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 52 eligible operators (44%). Final plan contains 11 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/extended.txt index 40230582bb..2cc00ffaf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/extended.txt @@ -1,47 +1,45 @@ CometNativeColumnarToRow +- CometSort - +- CometColumnarExchange - +- Project - +- BroadcastHashJoin - :- Filter - : +- HashAggregate - : +- CometNativeColumnarToRow - : +- CometColumnarExchange - : +- HashAggregate - : +- Project - : +- BroadcastHashJoin - : :- Project - : : +- BroadcastHashJoin - : : :- Project - : : : +- BroadcastHashJoin - : : : :- Filter - : : : : +- ColumnarToRow - : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] - : : : : +- SubqueryBroadcast - : : : : +- BroadcastExchange - : : : : +- CometNativeColumnarToRow - : : : : +- CometProject - : : : : +- CometFilter - : : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : : +- BroadcastExchange - : : : +- CometNativeColumnarToRow - : : : +- CometProject - : : : +- CometFilter - : : : +- CometNativeScan parquet spark_catalog.default.date_dim - : : +- BroadcastExchange - : : +- CometNativeColumnarToRow - : : +- CometProject - : : +- CometFilter - : : +- CometNativeScan parquet spark_catalog.default.store - : +- BroadcastExchange - : +- CometNativeColumnarToRow - : +- CometProject - : +- CometFilter - : +- CometNativeScan parquet spark_catalog.default.household_demographics - +- BroadcastExchange - +- CometNativeColumnarToRow - +- CometProject - +- CometFilter - +- CometNativeScan parquet spark_catalog.default.customer + +- CometExchange + +- CometProject + +- CometBroadcastHashJoin + :- CometFilter + : +- CometHashAggregate + : +- CometColumnarExchange + : +- HashAggregate + : +- Project + : +- BroadcastHashJoin + : :- Project + : : +- BroadcastHashJoin + : : :- Project + : : : +- BroadcastHashJoin + : : : :- Filter + : : : : +- ColumnarToRow + : : : : +- Scan parquet spark_catalog.default.store_sales [COMET: Native DataFusion scan does not support subqueries/dynamic pruning] + : : : : +- SubqueryBroadcast + : : : : +- BroadcastExchange + : : : : +- CometNativeColumnarToRow + : : : : +- CometProject + : : : : +- CometFilter + : : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : : +- BroadcastExchange + : : : +- CometNativeColumnarToRow + : : : +- CometProject + : : : +- CometFilter + : : : +- CometNativeScan parquet spark_catalog.default.date_dim + : : +- BroadcastExchange + : : +- CometNativeColumnarToRow + : : +- CometProject + : : +- CometFilter + : : +- CometNativeScan parquet spark_catalog.default.store + : +- BroadcastExchange + : +- CometNativeColumnarToRow + : +- CometProject + : +- CometFilter + : +- CometNativeScan parquet spark_catalog.default.household_demographics + +- CometBroadcastExchange + +- CometProject + +- CometFilter + +- CometNativeScan parquet spark_catalog.default.customer -Comet accelerated 18 out of 37 eligible operators (48%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 23 out of 37 eligible operators (62%). Final plan contains 6 transitions between Spark and Comet. \ No newline at end of file diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/extended.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/extended.txt index f2cdb50e03..d2f4f3dd01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/extended.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/extended.txt @@ -1,7 +1,7 @@ -TakeOrderedAndProject -+- Filter - +- HashAggregate - +- CometNativeColumnarToRow +CometNativeColumnarToRow ++- CometTakeOrderedAndProject + +- CometFilter + +- CometHashAggregate +- CometColumnarExchange +- HashAggregate +- Project @@ -65,4 +65,4 @@ TakeOrderedAndProject +- CometFilter +- CometNativeScan parquet spark_catalog.default.item -Comet accelerated 39 out of 58 eligible operators (67%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file +Comet accelerated 42 out of 58 eligible operators (72%). Final plan contains 8 transitions between Spark and Comet. \ No newline at end of file From 74ddc04e75528024146a965ce6b1627c040c1308 Mon Sep 17 00:00:00 2001 From: shekharrajak Date: Tue, 31 Mar 2026 23:05:21 +0530 Subject: [PATCH 10/12] Fix CI: remove stale aggregate filter check, add AQE test exclusions for all Spark versions --- dev/diffs/3.4.3.diff | 212 ++++++++------ dev/diffs/3.5.8.diff | 42 ++- dev/diffs/4.0.1.diff | 276 ++++++++++-------- .../apache/spark/sql/comet/operators.scala | 6 - 4 files changed, 323 insertions(+), 213 deletions(-) diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index 44f7601e36..3e935ed591 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -1,5 +1,5 @@ diff --git a/pom.xml b/pom.xml -index d3544881af1..377683b10c5 100644 +index d354488..377683b 100644 --- a/pom.xml +++ b/pom.xml @@ -148,6 +148,8 @@ @@ -38,7 +38,7 @@ index d3544881af1..377683b10c5 100644 diff --git a/sql/core/pom.xml b/sql/core/pom.xml -index b386d135da1..46449e3f3f1 100644 +index b386d13..46449e3 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -77,6 +77,10 @@ @@ -53,7 +53,7 @@ index b386d135da1..46449e3f3f1 100644