From 96b982693857f4fe4236dd208295f2378cb3d46a Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 24 Apr 2024 23:57:11 -0700 Subject: [PATCH 1/8] fix: CometShuffleExchangeExec logical link should be correct --- .../comet/CometSparkSessionExtensions.scala | 34 +- .../shuffle/CometShuffleExchangeExec.scala | 1 + .../shims/ShimCometShuffleExchangeExec.scala | 1 + .../approved-plans-v1_4/q16/explain.txt | 14 +- .../approved-plans-v1_4/q23b/explain.txt | 246 ++-- .../approved-plans-v1_4/q23b/simplified.txt | 21 +- .../approved-plans-v1_4/q24a/explain.txt | 22 +- .../approved-plans-v1_4/q24b/explain.txt | 22 +- .../approved-plans-v1_4/q28/explain.txt | 34 +- .../approved-plans-v1_4/q40/explain.txt | 12 +- .../approved-plans-v1_4/q41/explain.txt | 6 +- .../approved-plans-v1_4/q44/explain.txt | 237 ++-- .../approved-plans-v1_4/q44/simplified.txt | 14 +- .../approved-plans-v1_4/q54/explain.txt | 4 +- .../approved-plans-v1_4/q6/explain.txt | 12 +- .../approved-plans-v1_4/q64/explain.txt | 572 ++++++---- .../approved-plans-v1_4/q64/simplified.txt | 15 +- .../approved-plans-v1_4/q72/explain.txt | 6 +- .../approved-plans-v1_4/q75/explain.txt | 459 ++++---- .../approved-plans-v1_4/q75/simplified.txt | 19 +- .../approved-plans-v1_4/q78/explain.txt | 20 +- .../approved-plans-v1_4/q80/explain.txt | 32 +- .../approved-plans-v1_4/q9/explain.txt | 10 +- .../approved-plans-v1_4/q93/explain.txt | 8 +- .../approved-plans-v1_4/q94/explain.txt | 14 +- .../approved-plans-v1_4/q95/explain.txt | 327 +++--- .../approved-plans-v1_4/q95/simplified.txt | 15 +- .../approved-plans-v2_7/q24/explain.txt | 24 +- .../approved-plans-v2_7/q6/explain.txt | 12 +- .../approved-plans-v2_7/q64/explain.txt | 572 ++++++---- .../approved-plans-v2_7/q64/simplified.txt | 15 +- .../approved-plans-v2_7/q72/explain.txt | 6 +- .../approved-plans-v2_7/q75/explain.txt | 459 ++++---- .../approved-plans-v2_7/q75/simplified.txt | 19 +- .../approved-plans-v2_7/q78/explain.txt | 20 +- .../approved-plans-v2_7/q80a/explain.txt | 1008 +++++++++++++++-- .../approved-plans-v2_7/q80a/simplified.txt | 212 +++- .../apache/comet/exec/CometExecSuite.scala | 39 +- 38 files changed, 3065 insertions(+), 1498 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala index 8ef8cb83e8..1708544438 100644 --- a/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala +++ b/spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala @@ -741,9 +741,37 @@ class CometSparkSessionExtensions } // Set up logical links - newPlan = newPlan.transform { case op: CometExec => - op.originalPlan.logicalLink.foreach(op.setLogicalLink) - op + newPlan = newPlan.transform { + case op: CometExec => + if (op.originalPlan.logicalLink.isEmpty) { + op.unsetTagValue(SparkPlan.LOGICAL_PLAN_TAG) + op.unsetTagValue(SparkPlan.LOGICAL_PLAN_INHERITED_TAG) + } else { + op.originalPlan.logicalLink.foreach(op.setLogicalLink) + } + op + case op: CometShuffleExchangeExec => + // Original Spark shuffle exchange operator might have empty logical link. + // But the `setLogicalLink` call above on downstream operator of + // `CometShuffleExchangeExec` will set its logical link to the downstream + // operators which cause AQE behavior to be incorrect. So we need to unset + // the logical link here. + if (op.originalPlan.logicalLink.isEmpty) { + op.unsetTagValue(SparkPlan.LOGICAL_PLAN_TAG) + op.unsetTagValue(SparkPlan.LOGICAL_PLAN_INHERITED_TAG) + } else { + op.originalPlan.logicalLink.foreach(op.setLogicalLink) + } + op + + case op: CometBroadcastExchangeExec => + if (op.originalPlan.logicalLink.isEmpty) { + op.unsetTagValue(SparkPlan.LOGICAL_PLAN_TAG) + op.unsetTagValue(SparkPlan.LOGICAL_PLAN_INHERITED_TAG) + } else { + op.originalPlan.logicalLink.foreach(op.setLogicalLink) + } + op } // Convert native execution block by linking consecutive native operators. diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala index 232b6bf17f..faa52cd06b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala @@ -61,6 +61,7 @@ import org.apache.comet.shims.ShimCometShuffleExchangeExec case class CometShuffleExchangeExec( override val outputPartitioning: Partitioning, child: SparkPlan, + originalPlan: ShuffleExchangeLike, shuffleOrigin: ShuffleOrigin = ENSURE_REQUIREMENTS, shuffleType: ShuffleType = CometNativeShuffle, advisoryPartitionSize: Option[Long] = None) diff --git a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala index f89dbb8dba..6b4fad974f 100644 --- a/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala +++ b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala @@ -32,6 +32,7 @@ trait ShimCometShuffleExchangeExec { CometShuffleExchangeExec( s.outputPartitioning, s.child, + s, s.shuffleOrigin, shuffleType, advisoryPartitionSize) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt index 1762c8c6c8..61cd5f78b4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt @@ -63,7 +63,7 @@ Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_wareho (4) CometExchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#5, 5), Exchange hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (5) CometSort Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] @@ -84,7 +84,7 @@ Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order (9) CometExchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#10, 5), Exchange hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (10) CometSort Input [2]: [cs_warehouse_sk#9, cs_order_number#10] @@ -115,7 +115,7 @@ Arguments: [cr_order_number#12], [cr_order_number#12] (16) CometExchange Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#12, 5), Exchange hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (17) CometSort Input [1]: [cr_order_number#12] @@ -150,7 +150,7 @@ Input [1]: [d_date_sk#14] (24) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] (25) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_ship_date_sk#1] @@ -182,7 +182,7 @@ Input [1]: [ca_address_sk#16] (31) BroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] (32) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_ship_addr_sk#2] @@ -214,7 +214,7 @@ Input [1]: [cc_call_center_sk#18] (38) BroadcastExchange Input [1]: [cc_call_center_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] (39) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_call_center_sk#3] @@ -249,7 +249,7 @@ Results [3]: [sum#22, sum#23, count#25] (44) Exchange Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] (45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt index 86227b8324..f1b720650f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt @@ -1,6 +1,6 @@ == Physical Plan == -TakeOrderedAndProject (87) -+- Union (86) +TakeOrderedAndProject (98) ++- Union (97) :- * HashAggregate (62) : +- Exchange (61) : +- * HashAggregate (60) @@ -63,13 +63,13 @@ TakeOrderedAndProject (87) : : +- * HashAggregate (49) : : +- ReusedExchange (48) : +- ReusedExchange (57) - +- * HashAggregate (85) - +- Exchange (84) - +- * HashAggregate (83) - +- * Project (82) - +- * BroadcastHashJoin Inner BuildRight (81) - :- * Project (79) - : +- * BroadcastHashJoin Inner BuildRight (78) + +- * HashAggregate (96) + +- Exchange (95) + +- * HashAggregate (94) + +- * Project (93) + +- * BroadcastHashJoin Inner BuildRight (92) + :- * Project (90) + : +- * BroadcastHashJoin Inner BuildRight (89) : :- * SortMergeJoin LeftSemi (76) : : :- * Sort (70) : : : +- Exchange (69) @@ -84,8 +84,19 @@ TakeOrderedAndProject (87) : : +- * Filter (73) : : +- * HashAggregate (72) : : +- ReusedExchange (71) - : +- ReusedExchange (77) - +- ReusedExchange (80) + : +- BroadcastExchange (88) + : +- * SortMergeJoin LeftSemi (87) + : :- * ColumnarToRow (81) + : : +- CometSort (80) + : : +- CometExchange (79) + : : +- CometFilter (78) + : : +- CometScan parquet spark_catalog.default.customer (77) + : +- * Sort (86) + : +- * Project (85) + : +- * Filter (84) + : +- * HashAggregate (83) + : +- ReusedExchange (82) + +- ReusedExchange (91) (unknown) Scan parquet spark_catalog.default.catalog_sales @@ -118,7 +129,7 @@ Condition : isnotnull(ss_item_sk#7) (6) ColumnarToRow [codegen id : 3] Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -(7) ReusedExchange [Reuses operator id: 97] +(7) ReusedExchange [Reuses operator id: 108] Output [2]: [d_date_sk#10, d_date#11] (8) BroadcastHashJoin [codegen id : 3] @@ -302,7 +313,7 @@ Condition : isnotnull(c_customer_sk#33) (45) CometExchange Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] -Arguments: hashpartitioning(c_customer_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(c_customer_sk#33, 5), Exchange hashpartitioning(c_customer_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (46) CometSort Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] @@ -341,7 +352,7 @@ Join condition: None (54) BroadcastExchange Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (55) BroadcastHashJoin [codegen id : 16] Left keys [1]: [cs_bill_customer_sk#1] @@ -353,7 +364,7 @@ Join condition: None Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#34, c_last_name#35] Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#33, c_first_name#34, c_last_name#35] -(57) ReusedExchange [Reuses operator id: 92] +(57) ReusedExchange [Reuses operator id: 103] Output [1]: [d_date_sk#36] (58) BroadcastHashJoin [codegen id : 16] @@ -375,7 +386,7 @@ Results [4]: [c_last_name#35, c_first_name#34, sum#39, isEmpty#40] (61) Exchange Input [4]: [c_last_name#35, c_first_name#34, sum#39, isEmpty#40] -Arguments: hashpartitioning(c_last_name#35, c_first_name#34, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(c_last_name#35, c_first_name#34, 5), ENSURE_REQUIREMENTS, [plan_id=10] (62) HashAggregate [codegen id : 17] Input [4]: [c_last_name#35, c_first_name#34, sum#39, isEmpty#40] @@ -414,7 +425,7 @@ Input [5]: [ws_item_sk#43, ws_bill_customer_sk#44, ws_quantity#45, ws_list_price (69) Exchange Input [4]: [ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47] -Arguments: hashpartitioning(ws_bill_customer_sk#44, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(ws_bill_customer_sk#44, 5), ENSURE_REQUIREMENTS, [plan_id=11] (70) Sort [codegen id : 23] Input [4]: [ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47] @@ -448,64 +459,115 @@ Right keys [1]: [c_customer_sk#24] Join type: LeftSemi Join condition: None -(77) ReusedExchange [Reuses operator id: 54] +(unknown) Scan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(78) CometFilter +Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Condition : isnotnull(c_customer_sk#49) + +(79) CometExchange +Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: hashpartitioning(c_customer_sk#49, 5), Exchange hashpartitioning(c_customer_sk#49, 5), ENSURE_REQUIREMENTS, [plan_id=12], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] + +(80) CometSort +Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [c_customer_sk#49, c_first_name#50, c_last_name#51], [c_customer_sk#49 ASC NULLS FIRST] + +(81) ColumnarToRow [codegen id : 27] +Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] + +(82) ReusedExchange [Reuses operator id: 37] +Output [3]: [c_customer_sk#24, sum#27, isEmpty#28] + +(83) HashAggregate [codegen id : 30] +Input [3]: [c_customer_sk#24, sum#27, isEmpty#28] +Keys [1]: [c_customer_sk#24] +Functions [1]: [sum((cast(ss_quantity#21 as decimal(10,0)) * ss_sales_price#22))] +Aggregate Attributes [1]: [sum((cast(ss_quantity#21 as decimal(10,0)) * ss_sales_price#22))#29] +Results [2]: [c_customer_sk#24, sum((cast(ss_quantity#21 as decimal(10,0)) * ss_sales_price#22))#29 AS ssales#30] -(78) BroadcastHashJoin [codegen id : 33] +(84) Filter [codegen id : 30] +Input [2]: [c_customer_sk#24, ssales#30] +Condition : (isnotnull(ssales#30) AND (cast(ssales#30 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#31, [id=#32]))) + +(85) Project [codegen id : 30] +Output [1]: [c_customer_sk#24] +Input [2]: [c_customer_sk#24, ssales#30] + +(86) Sort [codegen id : 30] +Input [1]: [c_customer_sk#24] +Arguments: [c_customer_sk#24 ASC NULLS FIRST], false, 0 + +(87) SortMergeJoin [codegen id : 31] +Left keys [1]: [c_customer_sk#49] +Right keys [1]: [c_customer_sk#24] +Join type: LeftSemi +Join condition: None + +(88) BroadcastExchange +Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] + +(89) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ws_bill_customer_sk#44] Right keys [1]: [c_customer_sk#49] Join type: Inner Join condition: None -(79) Project [codegen id : 33] +(90) Project [codegen id : 33] Output [5]: [ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47, c_first_name#50, c_last_name#51] Input [7]: [ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47, c_customer_sk#49, c_first_name#50, c_last_name#51] -(80) ReusedExchange [Reuses operator id: 92] +(91) ReusedExchange [Reuses operator id: 103] Output [1]: [d_date_sk#52] -(81) BroadcastHashJoin [codegen id : 33] +(92) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ws_sold_date_sk#47] Right keys [1]: [d_date_sk#52] Join type: Inner Join condition: None -(82) Project [codegen id : 33] +(93) Project [codegen id : 33] Output [4]: [ws_quantity#45, ws_list_price#46, c_first_name#50, c_last_name#51] Input [6]: [ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47, c_first_name#50, c_last_name#51, d_date_sk#52] -(83) HashAggregate [codegen id : 33] +(94) HashAggregate [codegen id : 33] Input [4]: [ws_quantity#45, ws_list_price#46, c_first_name#50, c_last_name#51] Keys [2]: [c_last_name#51, c_first_name#50] Functions [1]: [partial_sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))] Aggregate Attributes [2]: [sum#53, isEmpty#54] Results [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -(84) Exchange +(95) Exchange Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(85) HashAggregate [codegen id : 34] +(96) HashAggregate [codegen id : 34] Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] Keys [2]: [c_last_name#51, c_first_name#50] Functions [1]: [sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))] Aggregate Attributes [1]: [sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))#57] Results [3]: [c_last_name#51, c_first_name#50, sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))#57 AS sales#58] -(86) Union +(97) Union -(87) TakeOrderedAndProject +(98) TakeOrderedAndProject Input [3]: [c_last_name#35, c_first_name#34, sales#42] Arguments: 100, [c_last_name#35 ASC NULLS FIRST, c_first_name#34 ASC NULLS FIRST, sales#42 ASC NULLS FIRST], [c_last_name#35, c_first_name#34, sales#42] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (92) -+- * ColumnarToRow (91) - +- CometProject (90) - +- CometFilter (89) - +- CometScan parquet spark_catalog.default.date_dim (88) +BroadcastExchange (103) ++- * ColumnarToRow (102) + +- CometProject (101) + +- CometFilter (100) + +- CometScan parquet spark_catalog.default.date_dim (99) (unknown) Scan parquet spark_catalog.default.date_dim @@ -515,27 +577,27 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(89) CometFilter +(100) CometFilter Input [3]: [d_date_sk#36, d_year#59, d_moy#60] Condition : ((((isnotnull(d_year#59) AND isnotnull(d_moy#60)) AND (d_year#59 = 2000)) AND (d_moy#60 = 2)) AND isnotnull(d_date_sk#36)) -(90) CometProject +(101) CometProject Input [3]: [d_date_sk#36, d_year#59, d_moy#60] Arguments: [d_date_sk#36], [d_date_sk#36] -(91) ColumnarToRow [codegen id : 1] +(102) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#36] -(92) BroadcastExchange +(103) BroadcastExchange Input [1]: [d_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=16] Subquery:2 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (97) -+- * ColumnarToRow (96) - +- CometProject (95) - +- CometFilter (94) - +- CometScan parquet spark_catalog.default.date_dim (93) +BroadcastExchange (108) ++- * ColumnarToRow (107) + +- CometProject (106) + +- CometFilter (105) + +- CometScan parquet spark_catalog.default.date_dim (104) (unknown) Scan parquet spark_catalog.default.date_dim @@ -545,37 +607,37 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(94) CometFilter +(105) CometFilter Input [3]: [d_date_sk#10, d_date#11, d_year#61] Condition : (d_year#61 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) -(95) CometProject +(106) CometProject Input [3]: [d_date_sk#10, d_date#11, d_year#61] Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] -(96) ColumnarToRow [codegen id : 1] +(107) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#10, d_date#11] -(97) BroadcastExchange +(108) BroadcastExchange Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] Subquery:3 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* HashAggregate (112) -+- Exchange (111) - +- * HashAggregate (110) - +- * HashAggregate (109) - +- Exchange (108) - +- * HashAggregate (107) - +- * Project (106) - +- * BroadcastHashJoin Inner BuildRight (105) - :- * Project (103) - : +- * BroadcastHashJoin Inner BuildRight (102) - : :- * ColumnarToRow (100) - : : +- CometFilter (99) - : : +- CometScan parquet spark_catalog.default.store_sales (98) - : +- ReusedExchange (101) - +- ReusedExchange (104) +* HashAggregate (123) ++- Exchange (122) + +- * HashAggregate (121) + +- * HashAggregate (120) + +- Exchange (119) + +- * HashAggregate (118) + +- * Project (117) + +- * BroadcastHashJoin Inner BuildRight (116) + :- * Project (114) + : +- * BroadcastHashJoin Inner BuildRight (113) + : :- * ColumnarToRow (111) + : : +- CometFilter (110) + : : +- CometScan parquet spark_catalog.default.store_sales (109) + : +- ReusedExchange (112) + +- ReusedExchange (115) (unknown) Scan parquet spark_catalog.default.store_sales @@ -586,81 +648,81 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#65), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(99) CometFilter +(110) CometFilter Input [4]: [ss_customer_sk#62, ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65] Condition : isnotnull(ss_customer_sk#62) -(100) ColumnarToRow [codegen id : 3] +(111) ColumnarToRow [codegen id : 3] Input [4]: [ss_customer_sk#62, ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65] -(101) ReusedExchange [Reuses operator id: 33] +(112) ReusedExchange [Reuses operator id: 33] Output [1]: [c_customer_sk#67] -(102) BroadcastHashJoin [codegen id : 3] +(113) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_customer_sk#62] Right keys [1]: [c_customer_sk#67] Join type: Inner Join condition: None -(103) Project [codegen id : 3] +(114) Project [codegen id : 3] Output [4]: [ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65, c_customer_sk#67] Input [5]: [ss_customer_sk#62, ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65, c_customer_sk#67] -(104) ReusedExchange [Reuses operator id: 117] +(115) ReusedExchange [Reuses operator id: 128] Output [1]: [d_date_sk#68] -(105) BroadcastHashJoin [codegen id : 3] +(116) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_sold_date_sk#65] Right keys [1]: [d_date_sk#68] Join type: Inner Join condition: None -(106) Project [codegen id : 3] +(117) Project [codegen id : 3] Output [3]: [ss_quantity#63, ss_sales_price#64, c_customer_sk#67] Input [5]: [ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65, c_customer_sk#67, d_date_sk#68] -(107) HashAggregate [codegen id : 3] +(118) HashAggregate [codegen id : 3] Input [3]: [ss_quantity#63, ss_sales_price#64, c_customer_sk#67] Keys [1]: [c_customer_sk#67] Functions [1]: [partial_sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))] Aggregate Attributes [2]: [sum#69, isEmpty#70] Results [3]: [c_customer_sk#67, sum#71, isEmpty#72] -(108) Exchange +(119) Exchange Input [3]: [c_customer_sk#67, sum#71, isEmpty#72] -Arguments: hashpartitioning(c_customer_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(c_customer_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=18] -(109) HashAggregate [codegen id : 4] +(120) HashAggregate [codegen id : 4] Input [3]: [c_customer_sk#67, sum#71, isEmpty#72] Keys [1]: [c_customer_sk#67] Functions [1]: [sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))] Aggregate Attributes [1]: [sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))#73] Results [1]: [sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))#73 AS csales#74] -(110) HashAggregate [codegen id : 4] +(121) HashAggregate [codegen id : 4] Input [1]: [csales#74] Keys: [] Functions [1]: [partial_max(csales#74)] Aggregate Attributes [1]: [max#75] Results [1]: [max#76] -(111) Exchange +(122) Exchange Input [1]: [max#76] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=19] -(112) HashAggregate [codegen id : 5] +(123) HashAggregate [codegen id : 5] Input [1]: [max#76] Keys: [] Functions [1]: [max(csales#74)] Aggregate Attributes [1]: [max(csales#74)#77] Results [1]: [max(csales#74)#77 AS tpcds_cmax#78] -Subquery:4 Hosting operator id = 98 Hosting Expression = ss_sold_date_sk#65 IN dynamicpruning#66 -BroadcastExchange (117) -+- * ColumnarToRow (116) - +- CometProject (115) - +- CometFilter (114) - +- CometScan parquet spark_catalog.default.date_dim (113) +Subquery:4 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#65 IN dynamicpruning#66 +BroadcastExchange (128) ++- * ColumnarToRow (127) + +- CometProject (126) + +- CometFilter (125) + +- CometScan parquet spark_catalog.default.date_dim (124) (unknown) Scan parquet spark_catalog.default.date_dim @@ -670,20 +732,20 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(114) CometFilter +(125) CometFilter Input [2]: [d_date_sk#68, d_year#79] Condition : (d_year#79 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#68)) -(115) CometProject +(126) CometProject Input [2]: [d_date_sk#68, d_year#79] Arguments: [d_date_sk#68], [d_date_sk#68] -(116) ColumnarToRow [codegen id : 1] +(127) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#68] -(117) BroadcastExchange +(128) BroadcastExchange Input [1]: [d_date_sk#68] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] Subquery:5 Hosting operator id = 50 Hosting Expression = ReusedSubquery Subquery scalar-subquery#31, [id=#32] @@ -691,4 +753,6 @@ Subquery:6 Hosting operator id = 63 Hosting Expression = ws_sold_date_sk#47 IN d Subquery:7 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#31, [id=#32] +Subquery:8 Hosting operator id = 84 Hosting Expression = ReusedSubquery Subquery scalar-subquery#31, [id=#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt index 9cbb568a9a..c53ac73588 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt @@ -181,6 +181,25 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] InputAdapter ReusedExchange [c_customer_sk,sum,isEmpty] #8 InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #13 + BroadcastExchange #17 + WholeStageCodegen (31) + SortMergeJoin [c_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (27) + ColumnarToRow + InputAdapter + CometSort [c_customer_sk] + CometExchange [c_customer_sk] #18 + CometFilter [c_customer_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + InputAdapter + WholeStageCodegen (30) + Sort [c_customer_sk] + Project [c_customer_sk] + Filter [ssales] + ReusedSubquery [tpcds_cmax] #3 + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] + InputAdapter + ReusedExchange [c_customer_sk,sum,isEmpty] #8 InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt index 6e03148d53..9e2b9f9f65 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt @@ -63,7 +63,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s (4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -86,7 +86,7 @@ Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8 (9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), Exchange hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -124,7 +124,7 @@ Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (18) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#3] @@ -152,7 +152,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (24) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] (25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -180,7 +180,7 @@ Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#2 (30) BroadcastExchange Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] (31) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_customer_sk#2] @@ -208,7 +208,7 @@ Input [3]: [ca_state#25, ca_zip#26, ca_country#27] (36) BroadcastExchange Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=8] (37) BroadcastHashJoin [codegen id : 5] Left keys [2]: [c_birth_country#24, s_zip#14] @@ -229,7 +229,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (40) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=9] (41) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] @@ -247,7 +247,7 @@ Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty# (43) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] (44) HashAggregate [codegen id : 7] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] @@ -348,7 +348,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (59) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] (60) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -395,7 +395,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (69) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=12] (70) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] @@ -413,7 +413,7 @@ Results [2]: [sum#44, count#45] (72) Exchange Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] (73) HashAggregate [codegen id : 7] Input [2]: [sum#44, count#45] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt index fb3113df00..f7aa1c2bc8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt @@ -63,7 +63,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s (4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -86,7 +86,7 @@ Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8 (9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), Exchange hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -124,7 +124,7 @@ Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (18) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#3] @@ -152,7 +152,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (24) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] (25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -180,7 +180,7 @@ Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#2 (30) BroadcastExchange Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] (31) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_customer_sk#2] @@ -208,7 +208,7 @@ Input [3]: [ca_state#25, ca_zip#26, ca_country#27] (36) BroadcastExchange Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=8] (37) BroadcastHashJoin [codegen id : 5] Left keys [2]: [c_birth_country#24, s_zip#14] @@ -229,7 +229,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (40) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=9] (41) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] @@ -247,7 +247,7 @@ Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty# (43) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] (44) HashAggregate [codegen id : 7] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] @@ -348,7 +348,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (59) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] (60) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -395,7 +395,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (69) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=12] (70) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] @@ -413,7 +413,7 @@ Results [2]: [sum#44, count#45] (72) Exchange Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] (73) HashAggregate [codegen id : 7] Input [2]: [sum#44, count#45] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt index db6d112e64..2b67b52a18 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt @@ -93,7 +93,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_li (5) CometExchange Input [4]: [ss_list_price#3, sum#6, count#7, count#8] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_list_price#3, 5), Exchange hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (6) ColumnarToRow [codegen id : 1] Input [4]: [ss_list_price#3, sum#6, count#7, count#8] @@ -114,7 +114,7 @@ Results [4]: [sum#6, count#7, count#8, count#12] (9) Exchange Input [4]: [sum#6, count#7, count#8, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] (10) HashAggregate [codegen id : 12] Input [4]: [sum#6, count#7, count#8, count#12] @@ -145,7 +145,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_l (15) CometExchange Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_list_price#18, 5), Exchange hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=4], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (16) ColumnarToRow [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] @@ -166,7 +166,7 @@ Results [4]: [sum#21, count#22, count#23, count#27] (19) Exchange Input [4]: [sum#21, count#22, count#23, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] (20) HashAggregate [codegen id : 3] Input [4]: [sum#21, count#22, count#23, count#27] @@ -177,7 +177,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal( (21) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] -Arguments: IdentityBroadcastMode, [plan_id=5] +Arguments: IdentityBroadcastMode, [plan_id=7] (22) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -205,7 +205,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_l (27) CometExchange Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_list_price#33, 5), Exchange hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=8], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (28) ColumnarToRow [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] @@ -226,7 +226,7 @@ Results [4]: [sum#36, count#37, count#38, count#42] (31) Exchange Input [4]: [sum#36, count#37, count#38, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] (32) HashAggregate [codegen id : 5] Input [4]: [sum#36, count#37, count#38, count#42] @@ -237,7 +237,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal( (33) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] -Arguments: IdentityBroadcastMode, [plan_id=8] +Arguments: IdentityBroadcastMode, [plan_id=11] (34) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -265,7 +265,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_l (39) CometExchange Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Arguments: hashpartitioning(ss_list_price#48, 5), Exchange hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=12], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (40) ColumnarToRow [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] @@ -286,7 +286,7 @@ Results [4]: [sum#51, count#52, count#53, count#57] (43) Exchange Input [4]: [sum#51, count#52, count#53, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=14] (44) HashAggregate [codegen id : 7] Input [4]: [sum#51, count#52, count#53, count#57] @@ -297,7 +297,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal( (45) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] -Arguments: IdentityBroadcastMode, [plan_id=11] +Arguments: IdentityBroadcastMode, [plan_id=15] (46) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -325,7 +325,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_l (51) CometExchange Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning(ss_list_price#63, 5), Exchange hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=16], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=17] (52) ColumnarToRow [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] @@ -346,7 +346,7 @@ Results [4]: [sum#66, count#67, count#68, count#72] (55) Exchange Input [4]: [sum#66, count#67, count#68, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] (56) HashAggregate [codegen id : 9] Input [4]: [sum#66, count#67, count#68, count#72] @@ -357,7 +357,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal( (57) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] -Arguments: IdentityBroadcastMode, [plan_id=14] +Arguments: IdentityBroadcastMode, [plan_id=19] (58) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -385,7 +385,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_l (63) CometExchange Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +Arguments: hashpartitioning(ss_list_price#78, 5), Exchange hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=20], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=21] (64) ColumnarToRow [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] @@ -406,7 +406,7 @@ Results [4]: [sum#81, count#82, count#83, count#87] (67) Exchange Input [4]: [sum#81, count#82, count#83, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=22] (68) HashAggregate [codegen id : 11] Input [4]: [sum#81, count#82, count#83, count#87] @@ -417,7 +417,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal( (69) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] -Arguments: IdentityBroadcastMode, [plan_id=17] +Arguments: IdentityBroadcastMode, [plan_id=23] (70) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt index 9f6036a23d..e0c36ee89c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt @@ -47,7 +47,7 @@ Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) (3) CometExchange Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), Exchange hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (4) CometSort Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] @@ -70,7 +70,7 @@ Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_item_sk#7, (8) CometExchange Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), Exchange hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (9) CometSort Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] @@ -104,7 +104,7 @@ Input [2]: [w_warehouse_sk#11, w_state#12] (16) BroadcastExchange Input [2]: [w_warehouse_sk#11, w_state#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (17) BroadcastHashJoin [codegen id : 4] Left keys [1]: [cs_warehouse_sk#1] @@ -136,7 +136,7 @@ Input [2]: [i_item_sk#13, i_item_id#14] (23) BroadcastExchange Input [2]: [i_item_sk#13, i_item_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (24) BroadcastHashJoin [codegen id : 4] Left keys [1]: [cs_item_sk#2] @@ -170,7 +170,7 @@ Results [6]: [w_state#12, i_item_id#14, sum#22, isEmpty#23, sum#24, isEmpty#25] (30) Exchange Input [6]: [w_state#12, i_item_id#14, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, [plan_id=7] (31) HashAggregate [codegen id : 5] Input [6]: [w_state#12, i_item_id#14, sum#22, isEmpty#23, sum#24, isEmpty#25] @@ -208,6 +208,6 @@ Input [2]: [d_date_sk#16, d_date#17] (36) BroadcastExchange Input [2]: [d_date_sk#16, d_date#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt index 2308990771..69f22dcdc1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt @@ -61,7 +61,7 @@ Functions [1]: [partial_count(1)] (9) CometExchange Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact#5, 5), Exchange hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometHashAggregate Input [2]: [i_manufact#5, count#9] @@ -81,7 +81,7 @@ Input [1]: [i_manufact#5] (14) BroadcastExchange Input [1]: [i_manufact#5] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] (15) BroadcastHashJoin [codegen id : 2] Left keys [1]: [i_manufact#2] @@ -102,7 +102,7 @@ Results [1]: [i_product_name#3] (18) Exchange Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [plan_id=4] (19) HashAggregate [codegen id : 3] Input [1]: [i_product_name#3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt index d7f13b274c..37d9690f93 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -TakeOrderedAndProject (33) -+- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * SortMergeJoin Inner (22) +TakeOrderedAndProject (40) ++- * Project (39) + +- * BroadcastHashJoin Inner BuildRight (38) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * Project (30) + : : +- * SortMergeJoin Inner (29) : : :- * Sort (14) : : : +- * Project (13) : : : +- * Filter (12) @@ -20,18 +20,25 @@ TakeOrderedAndProject (33) : : : +- CometProject (3) : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- * Sort (21) - : : +- * Project (20) - : : +- * Filter (19) - : : +- Window (18) - : : +- * ColumnarToRow (17) - : : +- CometSort (16) - : : +- ReusedExchange (15) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.item (24) - +- ReusedExchange (30) + : : +- * Sort (28) + : : +- * Project (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * ColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometFilter (21) + : : +- CometHashAggregate (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.store_sales (15) + : +- BroadcastExchange (34) + : +- * ColumnarToRow (33) + : +- CometFilter (32) + : +- CometScan parquet spark_catalog.default.item (31) + +- ReusedExchange (37) (unknown) Scan parquet spark_catalog.default.store_sales @@ -56,7 +63,7 @@ Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] (5) CometExchange Input [3]: [ss_item_sk#1, sum#5, count#6] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), Exchange hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (6) CometHashAggregate Input [3]: [ss_item_sk#1, sum#5, count#6] @@ -69,7 +76,7 @@ Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 (8) CometExchange Input [2]: [item_sk#7, rank_col#8] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (9) CometSort Input [2]: [item_sk#7, rank_col#8] @@ -94,129 +101,165 @@ Input [3]: [item_sk#7, rank_col#8, rnk#11] Input [2]: [item_sk#7, rnk#11] Arguments: [rnk#11 ASC NULLS FIRST], false, 0 -(15) ReusedExchange [Reuses operator id: 8] -Output [2]: [item_sk#12, rank_col#13] +(unknown) Scan parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#12, ss_store_sk#13, ss_net_profit#14, ss_sold_date_sk#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [ss_item_sk#12, ss_store_sk#13, ss_net_profit#14, ss_sold_date_sk#15] +Condition : (isnotnull(ss_store_sk#13) AND (ss_store_sk#13 = 4)) + +(17) CometProject +Input [4]: [ss_item_sk#12, ss_store_sk#13, ss_net_profit#14, ss_sold_date_sk#15] +Arguments: [ss_item_sk#12, ss_net_profit#14], [ss_item_sk#12, ss_net_profit#14] -(16) CometSort -Input [2]: [item_sk#12, rank_col#13] -Arguments: [item_sk#12, rank_col#13], [rank_col#13 DESC NULLS LAST] +(18) CometHashAggregate +Input [2]: [ss_item_sk#12, ss_net_profit#14] +Keys [1]: [ss_item_sk#12] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#14))] -(17) ColumnarToRow [codegen id : 3] -Input [2]: [item_sk#12, rank_col#13] +(19) CometExchange +Input [3]: [ss_item_sk#12, sum#16, count#17] +Arguments: hashpartitioning(ss_item_sk#12, 5), Exchange hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(18) Window -Input [2]: [item_sk#12, rank_col#13] -Arguments: [rank(rank_col#13) windowspecdefinition(rank_col#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#13 DESC NULLS LAST] +(20) CometHashAggregate +Input [3]: [ss_item_sk#12, sum#16, count#17] +Keys [1]: [ss_item_sk#12] +Functions [1]: [avg(UnscaledValue(ss_net_profit#14))] -(19) Filter [codegen id : 4] -Input [3]: [item_sk#12, rank_col#13, rnk#14] -Condition : ((rnk#14 < 11) AND isnotnull(item_sk#12)) +(21) CometFilter +Input [2]: [item_sk#18, rank_col#19] +Condition : (isnotnull(rank_col#19) AND (cast(rank_col#19 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) -(20) Project [codegen id : 4] -Output [2]: [item_sk#12, rnk#14] -Input [3]: [item_sk#12, rank_col#13, rnk#14] +(22) CometExchange +Input [2]: [item_sk#18, rank_col#19] +Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(21) Sort [codegen id : 4] -Input [2]: [item_sk#12, rnk#14] -Arguments: [rnk#14 ASC NULLS FIRST], false, 0 +(23) CometSort +Input [2]: [item_sk#18, rank_col#19] +Arguments: [item_sk#18, rank_col#19], [rank_col#19 DESC NULLS LAST] -(22) SortMergeJoin [codegen id : 7] +(24) ColumnarToRow [codegen id : 3] +Input [2]: [item_sk#18, rank_col#19] + +(25) Window +Input [2]: [item_sk#18, rank_col#19] +Arguments: [rank(rank_col#19) windowspecdefinition(rank_col#19 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#20], [rank_col#19 DESC NULLS LAST] + +(26) Filter [codegen id : 4] +Input [3]: [item_sk#18, rank_col#19, rnk#20] +Condition : ((rnk#20 < 11) AND isnotnull(item_sk#18)) + +(27) Project [codegen id : 4] +Output [2]: [item_sk#18, rnk#20] +Input [3]: [item_sk#18, rank_col#19, rnk#20] + +(28) Sort [codegen id : 4] +Input [2]: [item_sk#18, rnk#20] +Arguments: [rnk#20 ASC NULLS FIRST], false, 0 + +(29) SortMergeJoin [codegen id : 7] Left keys [1]: [rnk#11] -Right keys [1]: [rnk#14] +Right keys [1]: [rnk#20] Join type: Inner Join condition: None -(23) Project [codegen id : 7] -Output [3]: [item_sk#7, rnk#11, item_sk#12] -Input [4]: [item_sk#7, rnk#11, item_sk#12, rnk#14] +(30) Project [codegen id : 7] +Output [3]: [item_sk#7, rnk#11, item_sk#18] +Input [4]: [item_sk#7, rnk#11, item_sk#18, rnk#20] (unknown) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#15, i_product_name#16] +Output [2]: [i_item_sk#21, i_product_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(25) CometFilter -Input [2]: [i_item_sk#15, i_product_name#16] -Condition : isnotnull(i_item_sk#15) +(32) CometFilter +Input [2]: [i_item_sk#21, i_product_name#22] +Condition : isnotnull(i_item_sk#21) -(26) ColumnarToRow [codegen id : 5] -Input [2]: [i_item_sk#15, i_product_name#16] +(33) ColumnarToRow [codegen id : 5] +Input [2]: [i_item_sk#21, i_product_name#22] -(27) BroadcastExchange -Input [2]: [i_item_sk#15, i_product_name#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(34) BroadcastExchange +Input [2]: [i_item_sk#21, i_product_name#22] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -(28) BroadcastHashJoin [codegen id : 7] +(35) BroadcastHashJoin [codegen id : 7] Left keys [1]: [item_sk#7] -Right keys [1]: [i_item_sk#15] +Right keys [1]: [i_item_sk#21] Join type: Inner Join condition: None -(29) Project [codegen id : 7] -Output [3]: [rnk#11, item_sk#12, i_product_name#16] -Input [5]: [item_sk#7, rnk#11, item_sk#12, i_item_sk#15, i_product_name#16] +(36) Project [codegen id : 7] +Output [3]: [rnk#11, item_sk#18, i_product_name#22] +Input [5]: [item_sk#7, rnk#11, item_sk#18, i_item_sk#21, i_product_name#22] -(30) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#17, i_product_name#18] +(37) ReusedExchange [Reuses operator id: 34] +Output [2]: [i_item_sk#23, i_product_name#24] -(31) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [item_sk#12] -Right keys [1]: [i_item_sk#17] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [1]: [item_sk#18] +Right keys [1]: [i_item_sk#23] Join type: Inner Join condition: None -(32) Project [codegen id : 7] -Output [3]: [rnk#11, i_product_name#16 AS best_performing#19, i_product_name#18 AS worst_performing#20] -Input [5]: [rnk#11, item_sk#12, i_product_name#16, i_item_sk#17, i_product_name#18] +(39) Project [codegen id : 7] +Output [3]: [rnk#11, i_product_name#22 AS best_performing#25, i_product_name#24 AS worst_performing#26] +Input [5]: [rnk#11, item_sk#18, i_product_name#22, i_item_sk#23, i_product_name#24] -(33) TakeOrderedAndProject -Input [3]: [rnk#11, best_performing#19, worst_performing#20] -Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#19, worst_performing#20] +(40) TakeOrderedAndProject +Input [3]: [rnk#11, best_performing#25, worst_performing#26] +Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#25, worst_performing#26] ===== Subqueries ===== Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (40) -+- CometHashAggregate (39) - +- CometExchange (38) - +- CometHashAggregate (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.store_sales (34) +* ColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan parquet spark_catalog.default.store_sales (41) (unknown) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_addr_sk#21, ss_store_sk#22, ss_net_profit#23, ss_sold_date_sk#24] +Output [4]: [ss_addr_sk#27, ss_store_sk#28, ss_net_profit#29, ss_sold_date_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(35) CometFilter -Input [4]: [ss_addr_sk#21, ss_store_sk#22, ss_net_profit#23, ss_sold_date_sk#24] -Condition : ((isnotnull(ss_store_sk#22) AND (ss_store_sk#22 = 4)) AND isnull(ss_addr_sk#21)) +(42) CometFilter +Input [4]: [ss_addr_sk#27, ss_store_sk#28, ss_net_profit#29, ss_sold_date_sk#30] +Condition : ((isnotnull(ss_store_sk#28) AND (ss_store_sk#28 = 4)) AND isnull(ss_addr_sk#27)) + +(43) CometProject +Input [4]: [ss_addr_sk#27, ss_store_sk#28, ss_net_profit#29, ss_sold_date_sk#30] +Arguments: [ss_store_sk#28, ss_net_profit#29], [ss_store_sk#28, ss_net_profit#29] -(36) CometProject -Input [4]: [ss_addr_sk#21, ss_store_sk#22, ss_net_profit#23, ss_sold_date_sk#24] -Arguments: [ss_store_sk#22, ss_net_profit#23], [ss_store_sk#22, ss_net_profit#23] +(44) CometHashAggregate +Input [2]: [ss_store_sk#28, ss_net_profit#29] +Keys [1]: [ss_store_sk#28] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#29))] -(37) CometHashAggregate -Input [2]: [ss_store_sk#22, ss_net_profit#23] -Keys [1]: [ss_store_sk#22] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#23))] +(45) CometExchange +Input [3]: [ss_store_sk#28, sum#31, count#32] +Arguments: hashpartitioning(ss_store_sk#28, 5), Exchange hashpartitioning(ss_store_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=10], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(38) CometExchange -Input [3]: [ss_store_sk#22, sum#25, count#26] -Arguments: hashpartitioning(ss_store_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(46) CometHashAggregate +Input [3]: [ss_store_sk#28, sum#31, count#32] +Keys [1]: [ss_store_sk#28] +Functions [1]: [avg(UnscaledValue(ss_net_profit#29))] -(39) CometHashAggregate -Input [3]: [ss_store_sk#22, sum#25, count#26] -Keys [1]: [ss_store_sk#22] -Functions [1]: [avg(UnscaledValue(ss_net_profit#23))] +(47) ColumnarToRow [codegen id : 1] +Input [1]: [rank_col#33] -(40) ColumnarToRow [codegen id : 1] -Input [1]: [rank_col#27] +Subquery:2 Hosting operator id = 21 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt index bc065345c8..a1a70deade 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt @@ -46,13 +46,21 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] ColumnarToRow InputAdapter CometSort [rank_col] - ReusedExchange [item_sk,rank_col] #1 + CometExchange #4 + CometFilter [rank_col] + ReusedSubquery [rank_col] #1 + CometHashAggregate [ss_item_sk,sum,count] + CometExchange [ss_item_sk] #5 + CometHashAggregate [ss_item_sk,ss_net_profit] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_store_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter - BroadcastExchange #4 + BroadcastExchange #6 WholeStageCodegen (5) ColumnarToRow InputAdapter CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter - ReusedExchange [i_item_sk,i_product_name] #4 + ReusedExchange [i_item_sk,i_product_name] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt index 00c8f9c9b0..174a142f14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt @@ -420,7 +420,7 @@ Functions: [] (70) CometExchange Input [1]: [(d_month_seq + 1)#49] -Arguments: hashpartitioning((d_month_seq + 1)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Arguments: hashpartitioning((d_month_seq + 1)#49, 5), Exchange hashpartitioning((d_month_seq + 1)#49, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (71) CometHashAggregate Input [1]: [(d_month_seq + 1)#49] @@ -462,7 +462,7 @@ Functions: [] (77) CometExchange Input [1]: [(d_month_seq + 3)#53] -Arguments: hashpartitioning((d_month_seq + 3)#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning((d_month_seq + 3)#53, 5), Exchange hashpartitioning((d_month_seq + 3)#53, 5), ENSURE_REQUIREMENTS, [plan_id=13], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (78) CometHashAggregate Input [1]: [(d_month_seq + 3)#53] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt index 5f34aa3df9..0a63dcb589 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt @@ -156,7 +156,7 @@ Functions [1]: [partial_avg(UnscaledValue(i_current_price#13))] (25) CometExchange Input [3]: [i_category#14, sum#15, count#16] -Arguments: hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#14, 5), Exchange hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometHashAggregate Input [3]: [i_category#14, sum#15, count#16] @@ -172,7 +172,7 @@ Input [2]: [avg(i_current_price)#17, i_category#14] (29) BroadcastExchange Input [2]: [avg(i_current_price)#17, i_category#14] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] (30) BroadcastHashJoin [codegen id : 5] Left keys [1]: [i_category#12] @@ -186,7 +186,7 @@ Input [5]: [i_item_sk#10, i_current_price#11, i_category#12, avg(i_current_price (32) BroadcastExchange Input [1]: [i_item_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (33) BroadcastHashJoin [codegen id : 6] Left keys [1]: [ss_item_sk#5] @@ -207,7 +207,7 @@ Results [2]: [ca_state#2, count#19] (36) Exchange Input [2]: [ca_state#2, count#19] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] (37) HashAggregate [codegen id : 7] Input [2]: [ca_state#2, count#19] @@ -254,7 +254,7 @@ Input [1]: [d_date_sk#9] (44) BroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] Subquery:2 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#24, [id=#25] * ColumnarToRow (51) @@ -288,7 +288,7 @@ Functions: [] (49) CometExchange Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(d_month_seq#26, 5), Exchange hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (50) CometHashAggregate Input [1]: [d_month_seq#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt index c21eda6be3..8404fe47fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) +* Sort (192) ++- Exchange (191) + +- * Project (190) + +- * SortMergeJoin Inner (189) :- * Sort (108) : +- Exchange (107) : +- * HashAggregate (106) @@ -111,43 +111,43 @@ : +- CometProject (100) : +- CometFilter (99) : +- CometScan parquet spark_catalog.default.item (98) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * ColumnarToRow (126) - : : : : : : : : : : : : : : : +- CometProject (125) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (124) + +- * Sort (188) + +- Exchange (187) + +- * HashAggregate (186) + +- * HashAggregate (185) + +- * Project (184) + +- * BroadcastHashJoin Inner BuildRight (183) + :- * Project (181) + : +- * BroadcastHashJoin Inner BuildRight (180) + : :- * Project (178) + : : +- * BroadcastHashJoin Inner BuildRight (177) + : : :- * Project (175) + : : : +- * BroadcastHashJoin Inner BuildRight (174) + : : : :- * Project (172) + : : : : +- * BroadcastHashJoin Inner BuildRight (171) + : : : : :- * Project (169) + : : : : : +- * BroadcastHashJoin Inner BuildRight (168) + : : : : : :- * Project (166) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (165) + : : : : : : :- * Project (163) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (162) + : : : : : : : :- * Project (160) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (159) + : : : : : : : : :- * Project (157) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (156) + : : : : : : : : : :- * Project (154) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) + : : : : : : : : : : :- * Project (151) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) + : : : : : : : : : : : :- * Project (148) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) + : : : : : : : : : : : : :- * Project (145) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) + : : : : : : : : : : : : : :- * Project (142) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) + : : : : : : : : : : : : : : :- * ColumnarToRow (139) + : : : : : : : : : : : : : : : +- CometProject (138) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (137) : : : : : : : : : : : : : : : :- CometSort (118) : : : : : : : : : : : : : : : : +- CometExchange (117) : : : : : : : : : : : : : : : : +- CometProject (116) @@ -158,26 +158,39 @@ : : : : : : : : : : : : : : : : +- CometProject (114) : : : : : : : : : : : : : : : : +- CometFilter (113) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (112) - : : : : : : : : : : : : : : : +- CometSort (123) - : : : : : : : : : : : : : : : +- CometProject (122) - : : : : : : : : : : : : : : : +- CometFilter (121) - : : : : : : : : : : : : : : : +- CometHashAggregate (120) - : : : : : : : : : : : : : : : +- ReusedExchange (119) - : : : : : : : : : : : : : : +- ReusedExchange (127) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) + : : : : : : : : : : : : : : : +- CometSort (136) + : : : : : : : : : : : : : : : +- CometProject (135) + : : : : : : : : : : : : : : : +- CometFilter (134) + : : : : : : : : : : : : : : : +- CometHashAggregate (133) + : : : : : : : : : : : : : : : +- CometExchange (132) + : : : : : : : : : : : : : : : +- CometHashAggregate (131) + : : : : : : : : : : : : : : : +- CometProject (130) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (129) + : : : : : : : : : : : : : : : :- CometSort (123) + : : : : : : : : : : : : : : : : +- CometExchange (122) + : : : : : : : : : : : : : : : : +- CometProject (121) + : : : : : : : : : : : : : : : : +- CometFilter (120) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (119) + : : : : : : : : : : : : : : : +- CometSort (128) + : : : : : : : : : : : : : : : +- CometExchange (127) + : : : : : : : : : : : : : : : +- CometProject (126) + : : : : : : : : : : : : : : : +- CometFilter (125) + : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (124) + : : : : : : : : : : : : : : +- ReusedExchange (140) + : : : : : : : : : : : : : +- ReusedExchange (143) + : : : : : : : : : : : : +- ReusedExchange (146) + : : : : : : : : : : : +- ReusedExchange (149) + : : : : : : : : : : +- ReusedExchange (152) + : : : : : : : : : +- ReusedExchange (155) + : : : : : : : : +- ReusedExchange (158) + : : : : : : : +- ReusedExchange (161) + : : : : : : +- ReusedExchange (164) + : : : : : +- ReusedExchange (167) + : : : : +- ReusedExchange (170) + : : : +- ReusedExchange (173) + : : +- ReusedExchange (176) + : +- ReusedExchange (179) + +- ReusedExchange (182) (unknown) Scan parquet spark_catalog.default.store_sales @@ -222,7 +235,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), Exchange hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -245,7 +258,7 @@ Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_s (14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), Exchange hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -268,7 +281,7 @@ Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_ (19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), Exchange hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -290,7 +303,7 @@ Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum((( (24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, 5), Exchange hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -321,7 +334,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (31) ColumnarToRow [codegen id : 16] Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(32) ReusedExchange [Reuses operator id: 183] +(32) ReusedExchange [Reuses operator id: 196] Output [2]: [d_date_sk#32, d_year#33] (33) BroadcastHashJoin [codegen id : 16] @@ -350,7 +363,7 @@ Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] (38) BroadcastExchange Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (39) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_store_sk#6] @@ -378,7 +391,7 @@ Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_cu (44) BroadcastExchange Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] (45) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_customer_sk#2] @@ -406,7 +419,7 @@ Input [2]: [d_date_sk#43, d_year#44] (50) BroadcastExchange Input [2]: [d_date_sk#43, d_year#44] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] (51) BroadcastHashJoin [codegen id : 16] Left keys [1]: [c_first_sales_date_sk#42] @@ -447,7 +460,7 @@ Input [2]: [cd_demo_sk#47, cd_marital_status#48] (59) BroadcastExchange Input [2]: [cd_demo_sk#47, cd_marital_status#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] (60) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_cdemo_sk#3] @@ -488,7 +501,7 @@ Input [1]: [p_promo_sk#51] (68) BroadcastExchange Input [1]: [p_promo_sk#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] (69) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_promo_sk#7] @@ -516,7 +529,7 @@ Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] (74) BroadcastExchange Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] (75) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_hdemo_sk#4] @@ -557,7 +570,7 @@ Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59 (83) BroadcastExchange Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] (84) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_addr_sk#5] @@ -598,7 +611,7 @@ Input [1]: [ib_income_band_sk#66] (92) BroadcastExchange Input [1]: [ib_income_band_sk#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] (93) BroadcastHashJoin [codegen id : 16] Left keys [1]: [hd_income_band_sk#53] @@ -643,7 +656,7 @@ Input [2]: [i_item_sk#68, i_product_name#71] (102) BroadcastExchange Input [2]: [i_item_sk#68, i_product_name#71] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] (103) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_item_sk#1] @@ -671,7 +684,7 @@ Results [17]: [i_product_name#71 AS product_name#84, i_item_sk#68 AS item_sk#85, (107) Exchange Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] -Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=18] (108) Sort [codegen id : 17] Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] @@ -719,286 +732,347 @@ Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104 (117) CometExchange Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -Arguments: hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +Arguments: hashpartitioning(ss_item_sk#101, 5), Exchange hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=19], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=20] (118) CometSort Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101 ASC NULLS FIRST] -(119) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] +(unknown) Scan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct + +(120) CometFilter +Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] +Condition : (isnotnull(cs_item_sk#117) AND isnotnull(cs_order_number#118)) + +(121) CometProject +Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] +Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] + +(122) CometExchange +Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] +Arguments: hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), Exchange hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] + +(123) CometSort +Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] +Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117 ASC NULLS FIRST, cs_order_number#118 ASC NULLS FIRST] + +(unknown) Scan parquet spark_catalog.default.catalog_returns +Output [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(125) CometFilter +Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] +Condition : (isnotnull(cr_item_sk#121) AND isnotnull(cr_order_number#122)) + +(126) CometProject +Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] +Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] + +(127) CometExchange +Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), Exchange hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), ENSURE_REQUIREMENTS, [plan_id=23], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=24] + +(128) CometSort +Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121 ASC NULLS FIRST, cr_order_number#122 ASC NULLS FIRST] + +(129) CometSortMergeJoin +Left output [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] +Right output [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: [cs_item_sk#117, cs_order_number#118], [cr_item_sk#121, cr_order_number#122], Inner -(120) CometHashAggregate -Input [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] +(130) CometProject +Input [8]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] + +(131) CometHashAggregate +Input [5]: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] Keys [1]: [cs_item_sk#117] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#121)), sum(((cr_refunded_cash#122 + cr_reversed_charge#123) + cr_store_credit#124))] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#119)), partial_sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] + +(132) CometExchange +Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(cs_item_sk#117, 5), Exchange hashpartitioning(cs_item_sk#117, 5), ENSURE_REQUIREMENTS, [plan_id=25], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=26] -(121) CometFilter +(133) CometHashAggregate +Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] +Keys [1]: [cs_item_sk#117] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#119)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] + +(134) CometFilter Input [3]: [cs_item_sk#117, sale#30, refund#31] Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) -(122) CometProject +(135) CometProject Input [3]: [cs_item_sk#117, sale#30, refund#31] Arguments: [cs_item_sk#117], [cs_item_sk#117] -(123) CometSort +(136) CometSort Input [1]: [cs_item_sk#117] Arguments: [cs_item_sk#117], [cs_item_sk#117 ASC NULLS FIRST] -(124) CometSortMergeJoin +(137) CometSortMergeJoin Left output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Right output [1]: [cs_item_sk#117] Arguments: [ss_item_sk#101], [cs_item_sk#117], Inner -(125) CometProject +(138) CometProject Input [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, cs_item_sk#117] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(126) ColumnarToRow [codegen id : 33] +(139) ColumnarToRow [codegen id : 33] Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(127) ReusedExchange [Reuses operator id: 187] -Output [2]: [d_date_sk#125, d_year#126] +(140) ReusedExchange [Reuses operator id: 200] +Output [2]: [d_date_sk#130, d_year#131] -(128) BroadcastHashJoin [codegen id : 33] +(141) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_sold_date_sk#112] -Right keys [1]: [d_date_sk#125] +Right keys [1]: [d_date_sk#130] Join type: Inner Join condition: None -(129) Project [codegen id : 33] -Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126] -Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#125, d_year#126] +(142) Project [codegen id : 33] +Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131] +Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#130, d_year#131] -(130) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#127, s_store_name#128, s_zip#129] +(143) ReusedExchange [Reuses operator id: 38] +Output [3]: [s_store_sk#132, s_store_name#133, s_zip#134] -(131) BroadcastHashJoin [codegen id : 33] +(144) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_store_sk#106] -Right keys [1]: [s_store_sk#127] +Right keys [1]: [s_store_sk#132] Join type: Inner Join condition: None -(132) Project [codegen id : 33] -Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129] -Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_sk#127, s_store_name#128, s_zip#129] +(145) Project [codegen id : 33] +Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134] +Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_sk#132, s_store_name#133, s_zip#134] -(133) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] +(146) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] -(134) BroadcastHashJoin [codegen id : 33] +(147) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_customer_sk#102] -Right keys [1]: [c_customer_sk#130] +Right keys [1]: [c_customer_sk#135] Join type: Inner Join condition: None -(135) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] -Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] +(148) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] +Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] -(136) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#136, d_year#137] +(149) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#141, d_year#142] -(137) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_sales_date_sk#135] -Right keys [1]: [d_date_sk#136] +(150) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_sales_date_sk#140] +Right keys [1]: [d_date_sk#141] Join type: Inner Join condition: None -(138) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135, d_date_sk#136, d_year#137] +(151) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140, d_date_sk#141, d_year#142] -(139) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#138, d_year#139] +(152) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#143, d_year#144] -(140) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_shipto_date_sk#134] -Right keys [1]: [d_date_sk#138] +(153) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_shipto_date_sk#139] +Right keys [1]: [d_date_sk#143] Join type: Inner Join condition: None -(141) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137, d_date_sk#138, d_year#139] +(154) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142, d_date_sk#143, d_year#144] -(142) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#140, cd_marital_status#141] +(155) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] -(143) BroadcastHashJoin [codegen id : 33] +(156) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_cdemo_sk#103] -Right keys [1]: [cd_demo_sk#140] +Right keys [1]: [cd_demo_sk#145] Join type: Inner Join condition: None -(144) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_demo_sk#140, cd_marital_status#141] +(157) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_demo_sk#145, cd_marital_status#146] -(145) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#142, cd_marital_status#143] +(158) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#147, cd_marital_status#148] -(146) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_cdemo_sk#131] -Right keys [1]: [cd_demo_sk#142] +(159) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_cdemo_sk#136] +Right keys [1]: [cd_demo_sk#147] Join type: Inner -Join condition: NOT (cd_marital_status#141 = cd_marital_status#143) +Join condition: NOT (cd_marital_status#146 = cd_marital_status#148) -(147) Project [codegen id : 33] -Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] -Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141, cd_demo_sk#142, cd_marital_status#143] +(160) Project [codegen id : 33] +Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] +Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146, cd_demo_sk#147, cd_marital_status#148] -(148) ReusedExchange [Reuses operator id: 68] -Output [1]: [p_promo_sk#144] +(161) ReusedExchange [Reuses operator id: 68] +Output [1]: [p_promo_sk#149] -(149) BroadcastHashJoin [codegen id : 33] +(162) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_promo_sk#107] -Right keys [1]: [p_promo_sk#144] +Right keys [1]: [p_promo_sk#149] Join type: Inner Join condition: None -(150) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, p_promo_sk#144] +(163) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, p_promo_sk#149] -(151) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#145, hd_income_band_sk#146] +(164) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] -(152) BroadcastHashJoin [codegen id : 33] +(165) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_hdemo_sk#104] -Right keys [1]: [hd_demo_sk#145] +Right keys [1]: [hd_demo_sk#150] Join type: Inner Join condition: None -(153) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_demo_sk#145, hd_income_band_sk#146] +(166) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_demo_sk#150, hd_income_band_sk#151] -(154) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#147, hd_income_band_sk#148] +(167) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] -(155) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_hdemo_sk#132] -Right keys [1]: [hd_demo_sk#147] +(168) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_hdemo_sk#137] +Right keys [1]: [hd_demo_sk#152] Join type: Inner Join condition: None -(156) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148] -Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_demo_sk#147, hd_income_band_sk#148] +(169) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153] +Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_demo_sk#152, hd_income_band_sk#153] -(157) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] +(170) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(158) BroadcastHashJoin [codegen id : 33] +(171) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_addr_sk#105] -Right keys [1]: [ca_address_sk#149] +Right keys [1]: [ca_address_sk#154] Join type: Inner Join condition: None -(159) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] -Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] +(172) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(160) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(173) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -(161) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_addr_sk#133] -Right keys [1]: [ca_address_sk#154] +(174) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_addr_sk#138] +Right keys [1]: [ca_address_sk#159] Join type: Inner Join condition: None -(162) Project [codegen id : 33] -Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(175) Project [codegen id : 33] +Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -(163) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#159] +(176) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#164] -(164) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#146] -Right keys [1]: [ib_income_band_sk#159] +(177) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#164] Join type: Inner Join condition: None -(165) Project [codegen id : 33] -Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#159] +(178) Project [codegen id : 33] +Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#164] -(166) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#160] +(179) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#165] -(167) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#148] -Right keys [1]: [ib_income_band_sk#160] +(180) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#153] +Right keys [1]: [ib_income_band_sk#165] Join type: Inner Join condition: None -(168) Project [codegen id : 33] -Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#160] +(181) Project [codegen id : 33] +Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#165] -(169) ReusedExchange [Reuses operator id: 102] -Output [2]: [i_item_sk#161, i_product_name#162] +(182) ReusedExchange [Reuses operator id: 102] +Output [2]: [i_item_sk#166, i_product_name#167] -(170) BroadcastHashJoin [codegen id : 33] +(183) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_item_sk#101] -Right keys [1]: [i_item_sk#161] +Right keys [1]: [i_item_sk#166] Join type: Inner Join condition: None -(171) Project [codegen id : 33] -Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] +(184) Project [codegen id : 33] +Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] -(172) HashAggregate [codegen id : 33] -Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] -Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] +(185) HashAggregate [codegen id : 33] +Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] +Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#109)), partial_sum(UnscaledValue(ss_list_price#110)), partial_sum(UnscaledValue(ss_coupon_amt#111))] -Aggregate Attributes [4]: [count#72, sum#163, sum#164, sum#165] -Results [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] +Aggregate Attributes [4]: [count#72, sum#168, sum#169, sum#170] +Results [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] -(173) HashAggregate [codegen id : 33] -Input [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] -Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] +(186) HashAggregate [codegen id : 33] +Input [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] +Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#109)), sum(UnscaledValue(ss_list_price#110)), sum(UnscaledValue(ss_coupon_amt#111))] Aggregate Attributes [4]: [count(1)#80, sum(UnscaledValue(ss_wholesale_cost#109))#81, sum(UnscaledValue(ss_list_price#110))#82, sum(UnscaledValue(ss_coupon_amt#111))#83] -Results [8]: [i_item_sk#161 AS item_sk#169, s_store_name#128 AS store_name#170, s_zip#129 AS store_zip#171, d_year#126 AS syear#172, count(1)#80 AS cnt#173, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#174, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#175, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#176] +Results [8]: [i_item_sk#166 AS item_sk#174, s_store_name#133 AS store_name#175, s_zip#134 AS store_zip#176, d_year#131 AS syear#177, count(1)#80 AS cnt#178, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#179, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#180, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#181] -(174) Exchange -Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(187) Exchange +Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] +Arguments: hashpartitioning(item_sk#174, store_name#175, store_zip#176, 5), ENSURE_REQUIREMENTS, [plan_id=27] -(175) Sort [codegen id : 34] -Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -Arguments: [item_sk#169 ASC NULLS FIRST, store_name#170 ASC NULLS FIRST, store_zip#171 ASC NULLS FIRST], false, 0 +(188) Sort [codegen id : 34] +Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] +Arguments: [item_sk#174 ASC NULLS FIRST, store_name#175 ASC NULLS FIRST, store_zip#176 ASC NULLS FIRST], false, 0 -(176) SortMergeJoin [codegen id : 35] +(189) SortMergeJoin [codegen id : 35] Left keys [3]: [item_sk#85, store_name#86, store_zip#87] -Right keys [3]: [item_sk#169, store_name#170, store_zip#171] +Right keys [3]: [item_sk#174, store_name#175, store_zip#176] Join type: Inner -Join condition: (cnt#173 <= cnt#97) +Join condition: (cnt#178 <= cnt#97) -(177) Project [codegen id : 35] -Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] +(190) Project [codegen id : 35] +Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] +Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] -(178) Exchange -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=17] +(191) Exchange +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] +Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=28] -(179) Sort [codegen id : 36] -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST], true, 0 +(192) Sort [codegen id : 36] +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] +Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST], true, 0 ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (183) -+- * ColumnarToRow (182) - +- CometFilter (181) - +- CometScan parquet spark_catalog.default.date_dim (180) +BroadcastExchange (196) ++- * ColumnarToRow (195) + +- CometFilter (194) + +- CometScan parquet spark_catalog.default.date_dim (193) (unknown) Scan parquet spark_catalog.default.date_dim @@ -1008,40 +1082,40 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(181) CometFilter +(194) CometFilter Input [2]: [d_date_sk#32, d_year#33] Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) -(182) ColumnarToRow [codegen id : 1] +(195) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#32, d_year#33] -(183) BroadcastExchange +(196) BroadcastExchange Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=29] Subquery:2 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#112 IN dynamicpruning#113 -BroadcastExchange (187) -+- * ColumnarToRow (186) - +- CometFilter (185) - +- CometScan parquet spark_catalog.default.date_dim (184) +BroadcastExchange (200) ++- * ColumnarToRow (199) + +- CometFilter (198) + +- CometScan parquet spark_catalog.default.date_dim (197) (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#125, d_year#126] +Output [2]: [d_date_sk#130, d_year#131] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(185) CometFilter -Input [2]: [d_date_sk#125, d_year#126] -Condition : ((isnotnull(d_year#126) AND (d_year#126 = 2000)) AND isnotnull(d_date_sk#125)) +(198) CometFilter +Input [2]: [d_date_sk#130, d_year#131] +Condition : ((isnotnull(d_year#131) AND (d_year#131 = 2000)) AND isnotnull(d_date_sk#130)) -(186) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#125, d_year#126] +(199) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#130, d_year#131] -(187) BroadcastExchange -Input [2]: [d_date_sk#125, d_year#126] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] +(200) BroadcastExchange +Input [2]: [d_date_sk#130, d_year#131] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt index eda7db4e57..5b6f72526d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt @@ -221,7 +221,20 @@ WholeStageCodegen (36) CometProject [cs_item_sk] CometFilter [sale,refund] CometHashAggregate [cs_item_sk,sum,sum,isEmpty] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 + CometExchange [cs_item_sk] #22 + CometHashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number] + CometExchange [cs_item_sk,cs_order_number] #23 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #24 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #21 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt index c0d1f949a8..30e51d82d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt @@ -357,7 +357,7 @@ Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number# (62) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), Exchange hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=10], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (63) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] @@ -385,7 +385,7 @@ Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] (68) Exchange Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=12] (69) HashAggregate [codegen id : 14] Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] @@ -428,6 +428,6 @@ Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] (75) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index 3a29db88ef..62b89d9d95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -TakeOrderedAndProject (132) -+- * Project (131) - +- * SortMergeJoin Inner (130) +TakeOrderedAndProject (141) ++- * Project (140) + +- * SortMergeJoin Inner (139) :- * Sort (71) : +- Exchange (70) : +- * Filter (69) @@ -73,18 +73,18 @@ TakeOrderedAndProject (132) : +- CometProject (56) : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.web_returns (54) - +- * Sort (129) - +- Exchange (128) - +- * Filter (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- * HashAggregate (123) - +- Exchange (122) - +- * HashAggregate (121) - +- Union (120) - :- * Project (87) - : +- * SortMergeJoin LeftOuter (86) + +- * Sort (138) + +- Exchange (137) + +- * Filter (136) + +- * HashAggregate (135) + +- Exchange (134) + +- * HashAggregate (133) + +- * HashAggregate (132) + +- Exchange (131) + +- * HashAggregate (130) + +- Union (129) + :- * Project (90) + : +- * SortMergeJoin LeftOuter (89) : :- * Sort (82) : : +- Exchange (81) : : +- * Project (80) @@ -96,41 +96,50 @@ TakeOrderedAndProject (132) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) : : : +- ReusedExchange (75) : : +- ReusedExchange (78) - : +- * ColumnarToRow (85) - : +- CometSort (84) - : +- ReusedExchange (83) - :- * Project (103) - : +- * SortMergeJoin LeftOuter (102) - : :- * Sort (98) - : : +- Exchange (97) - : : +- * Project (96) - : : +- * BroadcastHashJoin Inner BuildRight (95) - : : :- * Project (93) - : : : +- * BroadcastHashJoin Inner BuildRight (92) - : : : :- * ColumnarToRow (90) - : : : : +- CometFilter (89) - : : : : +- CometScan parquet spark_catalog.default.store_sales (88) - : : : +- ReusedExchange (91) - : : +- ReusedExchange (94) - : +- * ColumnarToRow (101) - : +- CometSort (100) - : +- ReusedExchange (99) - +- * Project (119) - +- * SortMergeJoin LeftOuter (118) - :- * Sort (114) - : +- Exchange (113) - : +- * Project (112) - : +- * BroadcastHashJoin Inner BuildRight (111) - : :- * Project (109) - : : +- * BroadcastHashJoin Inner BuildRight (108) - : : :- * ColumnarToRow (106) - : : : +- CometFilter (105) - : : : +- CometScan parquet spark_catalog.default.web_sales (104) - : : +- ReusedExchange (107) - : +- ReusedExchange (110) - +- * ColumnarToRow (117) - +- CometSort (116) - +- ReusedExchange (115) + : +- * ColumnarToRow (88) + : +- CometSort (87) + : +- CometExchange (86) + : +- CometProject (85) + : +- CometFilter (84) + : +- CometScan parquet spark_catalog.default.catalog_returns (83) + :- * Project (109) + : +- * SortMergeJoin LeftOuter (108) + : :- * Sort (101) + : : +- Exchange (100) + : : +- * Project (99) + : : +- * BroadcastHashJoin Inner BuildRight (98) + : : :- * Project (96) + : : : +- * BroadcastHashJoin Inner BuildRight (95) + : : : :- * ColumnarToRow (93) + : : : : +- CometFilter (92) + : : : : +- CometScan parquet spark_catalog.default.store_sales (91) + : : : +- ReusedExchange (94) + : : +- ReusedExchange (97) + : +- * ColumnarToRow (107) + : +- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometFilter (103) + : +- CometScan parquet spark_catalog.default.store_returns (102) + +- * Project (128) + +- * SortMergeJoin LeftOuter (127) + :- * Sort (120) + : +- Exchange (119) + : +- * Project (118) + : +- * BroadcastHashJoin Inner BuildRight (117) + : :- * Project (115) + : : +- * BroadcastHashJoin Inner BuildRight (114) + : : :- * ColumnarToRow (112) + : : : +- CometFilter (111) + : : : +- CometScan parquet spark_catalog.default.web_sales (110) + : : +- ReusedExchange (113) + : +- ReusedExchange (116) + +- * ColumnarToRow (126) + +- CometSort (125) + +- CometExchange (124) + +- CometProject (123) + +- CometFilter (122) + +- CometScan parquet spark_catalog.default.web_returns (121) (unknown) Scan parquet spark_catalog.default.catalog_sales @@ -180,7 +189,7 @@ Join condition: None Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -(11) ReusedExchange [Reuses operator id: 136] +(11) ReusedExchange [Reuses operator id: 145] Output [2]: [d_date_sk#13, d_year#14] (12) BroadcastHashJoin [codegen id : 3] @@ -218,7 +227,7 @@ Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_ (19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), Exchange hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -265,7 +274,7 @@ Join condition: None Output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -(30) ReusedExchange [Reuses operator id: 136] +(30) ReusedExchange [Reuses operator id: 145] Output [2]: [d_date_sk#33, d_year#34] (31) BroadcastHashJoin [codegen id : 9] @@ -280,7 +289,7 @@ Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_pr (33) Exchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=5] (34) Sort [codegen id : 10] Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -303,7 +312,7 @@ Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return (38) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), Exchange hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (39) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -350,7 +359,7 @@ Join condition: None Output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -(49) ReusedExchange [Reuses operator id: 136] +(49) ReusedExchange [Reuses operator id: 145] Output [2]: [d_date_sk#53, d_year#54] (50) BroadcastHashJoin [codegen id : 15] @@ -365,7 +374,7 @@ Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_pri (52) Exchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] (53) Sort [codegen id : 16] Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -388,7 +397,7 @@ Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_ (57) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), Exchange hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (58) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -418,7 +427,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (64) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=11] (65) HashAggregate [codegen id : 20] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -436,7 +445,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (67) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=12] (68) HashAggregate [codegen id : 21] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] @@ -451,7 +460,7 @@ Condition : isnotnull(sales_cnt#68) (70) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=13] (71) Sort [codegen id : 22] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] @@ -485,7 +494,7 @@ Join condition: None Output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Input [10]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(78) ReusedExchange [Reuses operator id: 140] +(78) ReusedExchange [Reuses operator id: 149] Output [2]: [d_date_sk#81, d_year#82] (79) BroadcastHashJoin [codegen id : 25] @@ -500,241 +509,289 @@ Input [11]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_pri (81) Exchange Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=14] (82) Sort [codegen id : 26] Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Arguments: [cs_order_number#71 ASC NULLS FIRST, cs_item_sk#70 ASC NULLS FIRST], false, 0 -(83) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] +(unknown) Scan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(84) CometFilter +Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] +Condition : (isnotnull(cr_order_number#84) AND isnotnull(cr_item_sk#83)) -(84) CometSort +(85) CometProject +Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] +Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] + +(86) CometExchange +Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] +Arguments: hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), Exchange hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] + +(87) CometSort Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_order_number#84 ASC NULLS FIRST, cr_item_sk#83 ASC NULLS FIRST] -(85) ColumnarToRow [codegen id : 27] +(88) ColumnarToRow [codegen id : 27] Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(86) SortMergeJoin [codegen id : 28] +(89) SortMergeJoin [codegen id : 28] Left keys [2]: [cs_order_number#71, cs_item_sk#70] Right keys [2]: [cr_order_number#84, cr_item_sk#83] Join type: LeftOuter Join condition: None -(87) Project [codegen id : 28] +(90) Project [codegen id : 28] Output [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, (cs_quantity#72 - coalesce(cr_return_quantity#85, 0)) AS sales_cnt#20, (cs_ext_sales_price#73 - coalesce(cr_return_amount#86, 0.00)) AS sales_amt#21] Input [13]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82, cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] (unknown) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +Output [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#91), dynamicpruningexpression(ss_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ss_sold_date_sk#92), dynamicpruningexpression(ss_sold_date_sk#92 IN dynamicpruning#93)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(89) CometFilter -Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -Condition : isnotnull(ss_item_sk#87) +(92) CometFilter +Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] +Condition : isnotnull(ss_item_sk#88) -(90) ColumnarToRow [codegen id : 31] -Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +(93) ColumnarToRow [codegen id : 31] +Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] -(91) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +(94) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] -(92) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_item_sk#87] -Right keys [1]: [i_item_sk#93] +(95) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_item_sk#88] +Right keys [1]: [i_item_sk#94] Join type: Inner Join condition: None -(93) Project [codegen id : 31] -Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Input [10]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +(96) Project [codegen id : 31] +Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] +Input [10]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] -(94) ReusedExchange [Reuses operator id: 140] -Output [2]: [d_date_sk#98, d_year#99] +(97) ReusedExchange [Reuses operator id: 149] +Output [2]: [d_date_sk#99, d_year#100] -(95) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_sold_date_sk#91] -Right keys [1]: [d_date_sk#98] +(98) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_sold_date_sk#92] +Right keys [1]: [d_date_sk#99] Join type: Inner Join condition: None -(96) Project [codegen id : 31] -Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_date_sk#98, d_year#99] +(99) Project [codegen id : 31] +Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] +Input [11]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_date_sk#99, d_year#100] -(97) Exchange -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(100) Exchange +Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] +Arguments: hashpartitioning(ss_ticket_number#89, ss_item_sk#88, 5), ENSURE_REQUIREMENTS, [plan_id=17] -(98) Sort [codegen id : 32] -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: [ss_ticket_number#88 ASC NULLS FIRST, ss_item_sk#87 ASC NULLS FIRST], false, 0 +(101) Sort [codegen id : 32] +Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] +Arguments: [ss_ticket_number#89 ASC NULLS FIRST, ss_item_sk#88 ASC NULLS FIRST], false, 0 -(99) ReusedExchange [Reuses operator id: 38] -Output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +(unknown) Scan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(103) CometFilter +Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] +Condition : (isnotnull(sr_ticket_number#102) AND isnotnull(sr_item_sk#101)) -(100) CometSort -Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103], [sr_ticket_number#101 ASC NULLS FIRST, sr_item_sk#100 ASC NULLS FIRST] +(104) CometProject +Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] +Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] -(101) ColumnarToRow [codegen id : 33] -Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +(105) CometExchange +Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +Arguments: hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), Exchange hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=18], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=19] -(102) SortMergeJoin [codegen id : 34] -Left keys [2]: [ss_ticket_number#88, ss_item_sk#87] -Right keys [2]: [sr_ticket_number#101, sr_item_sk#100] +(106) CometSort +Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_ticket_number#102 ASC NULLS FIRST, sr_item_sk#101 ASC NULLS FIRST] + +(107) ColumnarToRow [codegen id : 33] +Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] + +(108) SortMergeJoin [codegen id : 34] +Left keys [2]: [ss_ticket_number#89, ss_item_sk#88] +Right keys [2]: [sr_ticket_number#102, sr_item_sk#101] Join type: LeftOuter Join condition: None -(103) Project [codegen id : 34] -Output [7]: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, (ss_quantity#89 - coalesce(sr_return_quantity#102, 0)) AS sales_cnt#40, (ss_ext_sales_price#90 - coalesce(sr_return_amt#103, 0.00)) AS sales_amt#41] -Input [13]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99, sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +(109) Project [codegen id : 34] +Output [7]: [d_year#100, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, (ss_quantity#90 - coalesce(sr_return_quantity#103, 0)) AS sales_cnt#40, (ss_ext_sales_price#91 - coalesce(sr_return_amt#104, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100, sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] (unknown) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] +Output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#108), dynamicpruningexpression(ws_sold_date_sk#108 IN dynamicpruning#109)] +PartitionFilters: [isnotnull(ws_sold_date_sk#110), dynamicpruningexpression(ws_sold_date_sk#110 IN dynamicpruning#111)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(105) CometFilter -Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] -Condition : isnotnull(ws_item_sk#104) +(111) CometFilter +Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +Condition : isnotnull(ws_item_sk#106) -(106) ColumnarToRow [codegen id : 37] -Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] +(112) ColumnarToRow [codegen id : 37] +Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -(107) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] +(113) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -(108) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_item_sk#104] -Right keys [1]: [i_item_sk#110] +(114) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_item_sk#106] +Right keys [1]: [i_item_sk#112] Join type: Inner Join condition: None -(109) Project [codegen id : 37] -Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] -Input [10]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] +(115) Project [codegen id : 37] +Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +Input [10]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -(110) ReusedExchange [Reuses operator id: 140] -Output [2]: [d_date_sk#115, d_year#116] +(116) ReusedExchange [Reuses operator id: 149] +Output [2]: [d_date_sk#117, d_year#118] -(111) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_sold_date_sk#108] -Right keys [1]: [d_date_sk#115] +(117) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_sold_date_sk#110] +Right keys [1]: [d_date_sk#117] Join type: Inner Join condition: None -(112) Project [codegen id : 37] -Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Input [11]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_date_sk#115, d_year#116] +(118) Project [codegen id : 37] +Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Input [11]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_date_sk#117, d_year#118] + +(119) Exchange +Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Arguments: hashpartitioning(ws_order_number#107, ws_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(120) Sort [codegen id : 38] +Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Arguments: [ws_order_number#107 ASC NULLS FIRST, ws_item_sk#106 ASC NULLS FIRST], false, 0 + +(unknown) Scan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct -(113) Exchange -Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(122) CometFilter +Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] +Condition : (isnotnull(wr_order_number#120) AND isnotnull(wr_item_sk#119)) -(114) Sort [codegen id : 38] -Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Arguments: [ws_order_number#105 ASC NULLS FIRST, ws_item_sk#104 ASC NULLS FIRST], false, 0 +(123) CometProject +Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] +Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -(115) ReusedExchange [Reuses operator id: 57] -Output [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +(124) CometExchange +Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +Arguments: hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), Exchange hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] -(116) CometSort -Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -Arguments: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120], [wr_order_number#118 ASC NULLS FIRST, wr_item_sk#117 ASC NULLS FIRST] +(125) CometSort +Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_order_number#120 ASC NULLS FIRST, wr_item_sk#119 ASC NULLS FIRST] -(117) ColumnarToRow [codegen id : 39] -Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +(126) ColumnarToRow [codegen id : 39] +Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -(118) SortMergeJoin [codegen id : 40] -Left keys [2]: [ws_order_number#105, ws_item_sk#104] -Right keys [2]: [wr_order_number#118, wr_item_sk#117] +(127) SortMergeJoin [codegen id : 40] +Left keys [2]: [ws_order_number#107, ws_item_sk#106] +Right keys [2]: [wr_order_number#120, wr_item_sk#119] Join type: LeftOuter Join condition: None -(119) Project [codegen id : 40] -Output [7]: [d_year#116, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, (ws_quantity#106 - coalesce(wr_return_quantity#119, 0)) AS sales_cnt#60, (ws_ext_sales_price#107 - coalesce(wr_return_amt#120, 0.00)) AS sales_amt#61] -Input [13]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116, wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +(128) Project [codegen id : 40] +Output [7]: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, (ws_quantity#108 - coalesce(wr_return_quantity#121, 0)) AS sales_cnt#60, (ws_ext_sales_price#109 - coalesce(wr_return_amt#122, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118, wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -(120) Union +(129) Union -(121) HashAggregate [codegen id : 41] +(130) HashAggregate [codegen id : 41] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(122) Exchange +(131) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=23] -(123) HashAggregate [codegen id : 42] +(132) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(124) HashAggregate [codegen id : 42] +(133) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -Aggregate Attributes [2]: [sum#62, sum#121] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] +Aggregate Attributes [2]: [sum#62, sum#124] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] -(125) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(134) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=24] -(126) HashAggregate [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] +(135) HashAggregate [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Aggregate Attributes [2]: [sum(sales_cnt#20)#66, sum(UnscaledValue(sales_amt#21))#67] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#123, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#124] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#126, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#127] -(127) Filter [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -Condition : isnotnull(sales_cnt#123) +(136) Filter [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +Condition : isnotnull(sales_cnt#126) -(128) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(137) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=25] -(129) Sort [codegen id : 44] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +(138) Sort [codegen id : 44] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] Arguments: [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST], false, 0 -(130) SortMergeJoin [codegen id : 45] +(139) SortMergeJoin [codegen id : 45] Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Right keys [4]: [i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Join type: Inner -Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#123 as decimal(17,2))) < 0.90000000000000000000) +Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#126 as decimal(17,2))) < 0.90000000000000000000) -(131) Project [codegen id : 45] -Output [10]: [d_year#82 AS prev_year#125, d_year#14 AS year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#123 AS prev_yr_cnt#127, sales_cnt#68 AS curr_yr_cnt#128, (sales_cnt#68 - sales_cnt#123) AS sales_cnt_diff#129, (sales_amt#69 - sales_amt#124) AS sales_amt_diff#130] -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +(140) Project [codegen id : 45] +Output [10]: [d_year#82 AS prev_year#128, d_year#14 AS year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#126 AS prev_yr_cnt#130, sales_cnt#68 AS curr_yr_cnt#131, (sales_cnt#68 - sales_cnt#126) AS sales_cnt_diff#132, (sales_amt#69 - sales_amt#127) AS sales_amt_diff#133] +Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] -(132) TakeOrderedAndProject -Input [10]: [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] -Arguments: 100, [sales_cnt_diff#129 ASC NULLS FIRST], [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] +(141) TakeOrderedAndProject +Input [10]: [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] +Arguments: 100, [sales_cnt_diff#132 ASC NULLS FIRST], [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (136) -+- * ColumnarToRow (135) - +- CometFilter (134) - +- CometScan parquet spark_catalog.default.date_dim (133) +BroadcastExchange (145) ++- * ColumnarToRow (144) + +- CometFilter (143) + +- CometScan parquet spark_catalog.default.date_dim (142) (unknown) Scan parquet spark_catalog.default.date_dim @@ -744,26 +801,26 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(134) CometFilter +(143) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(135) ColumnarToRow [codegen id : 1] +(144) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -(136) BroadcastExchange +(145) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=26] Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 -BroadcastExchange (140) -+- * ColumnarToRow (139) - +- CometFilter (138) - +- CometScan parquet spark_catalog.default.date_dim (137) +BroadcastExchange (149) ++- * ColumnarToRow (148) + +- CometFilter (147) + +- CometScan parquet spark_catalog.default.date_dim (146) (unknown) Scan parquet spark_catalog.default.date_dim @@ -773,19 +830,19 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(138) CometFilter +(147) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(139) ColumnarToRow [codegen id : 1] +(148) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -(140) BroadcastExchange +(149) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=27] -Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#75 +Subquery:5 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#92 IN dynamicpruning#75 -Subquery:6 Hosting operator id = 104 Hosting Expression = ws_sold_date_sk#108 IN dynamicpruning#75 +Subquery:6 Hosting operator id = 110 Hosting Expression = ws_sold_date_sk#110 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt index 837398c8f1..6e9e0ea9e3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt @@ -178,7 +178,10 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [cr_order_number,cr_item_sk] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometExchange [cr_order_number,cr_item_sk] #17 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_order_number,cr_item_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (34) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] @@ -186,7 +189,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat WholeStageCodegen (32) Sort [ss_ticket_number,ss_item_sk] InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 + Exchange [ss_ticket_number,ss_item_sk] #18 WholeStageCodegen (31) Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] @@ -206,7 +209,10 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [sr_ticket_number,sr_item_sk] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometExchange [sr_ticket_number,sr_item_sk] #19 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_ticket_number,sr_item_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (40) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] @@ -214,7 +220,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat WholeStageCodegen (38) Sort [ws_order_number,ws_item_sk] InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 + Exchange [ws_order_number,ws_item_sk] #20 WholeStageCodegen (37) Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] @@ -234,4 +240,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [wr_order_number,wr_item_sk] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 + CometExchange [wr_order_number,wr_item_sk] #21 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_order_number,wr_item_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt index caab92168f..25ac63aca3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt @@ -82,7 +82,7 @@ Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) (3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -105,7 +105,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number# (8) CometExchange Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), Exchange hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (9) CometSort Input [2]: [sr_item_sk#9, sr_ticket_number#10] @@ -149,7 +149,7 @@ Results [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] (18) Exchange Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=5] (19) HashAggregate [codegen id : 3] Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] @@ -176,7 +176,7 @@ Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) (23) CometExchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), Exchange hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (24) CometSort Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] @@ -199,7 +199,7 @@ Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number# (28) CometExchange Input [2]: [wr_item_sk#35, wr_order_number#36] -Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), Exchange hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=8], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (29) CometSort Input [2]: [wr_item_sk#35, wr_order_number#36] @@ -243,7 +243,7 @@ Results [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, (38) Exchange Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=10] (39) HashAggregate [codegen id : 6] Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] @@ -284,7 +284,7 @@ Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) (46) CometExchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), Exchange hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (47) CometSort Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] @@ -307,7 +307,7 @@ Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number# (51) CometExchange Input [2]: [cr_item_sk#62, cr_order_number#63] -Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), Exchange hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, [plan_id=13], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (52) CometSort Input [2]: [cr_item_sk#62, cr_order_number#63] @@ -351,7 +351,7 @@ Results [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, (61) Exchange Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=15] (62) HashAggregate [codegen id : 10] Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] @@ -407,7 +407,7 @@ Input [2]: [d_date_sk#12, d_year#13] (71) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] Subquery:2 Hosting operator id = 21 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt index dfd123a1d9..39b431840c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt @@ -119,7 +119,7 @@ Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnul (3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -142,7 +142,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] (8) CometExchange Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), Exchange hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (9) CometSort Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] @@ -189,7 +189,7 @@ Input [2]: [s_store_sk#15, s_store_id#16] (19) BroadcastExchange Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (20) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#2] @@ -221,7 +221,7 @@ Input [1]: [i_item_sk#17] (26) BroadcastExchange Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (27) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -253,7 +253,7 @@ Input [1]: [p_promo_sk#19] (33) BroadcastExchange Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_promo_sk#3] @@ -274,7 +274,7 @@ Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] (37) Exchange Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=8] (38) HashAggregate [codegen id : 6] Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] @@ -297,7 +297,7 @@ Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND (41) CometExchange Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (42) CometSort Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] @@ -320,7 +320,7 @@ Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss# (46) CometExchange Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), Exchange hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (47) CometSort Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] @@ -367,7 +367,7 @@ Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] (57) BroadcastExchange Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] (58) BroadcastHashJoin [codegen id : 11] Left keys [1]: [cs_catalog_page_sk#39] @@ -414,7 +414,7 @@ Results [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty (67) Exchange Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] -Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=14] (68) HashAggregate [codegen id : 12] Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] @@ -437,7 +437,7 @@ Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isn (71) CometExchange Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] (72) CometSort Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] @@ -460,7 +460,7 @@ Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] (76) CometExchange Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), Exchange hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, [plan_id=17], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] (77) CometSort Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] @@ -507,7 +507,7 @@ Input [2]: [web_site_sk#89, web_site_id#90] (87) BroadcastExchange Input [2]: [web_site_sk#89, web_site_id#90] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] (88) BroadcastHashJoin [codegen id : 17] Left keys [1]: [ws_web_site_sk#76] @@ -554,7 +554,7 @@ Results [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (97) Exchange Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=20] (98) HashAggregate [codegen id : 18] Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] @@ -578,7 +578,7 @@ Results [9]: [channel#111, id#112, spark_grouping_id#113, sum#120, isEmpty#121, (102) Exchange Input [9]: [channel#111, id#112, spark_grouping_id#113, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] -Arguments: hashpartitioning(channel#111, id#112, spark_grouping_id#113, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(channel#111, id#112, spark_grouping_id#113, 5), ENSURE_REQUIREMENTS, [plan_id=21] (103) HashAggregate [codegen id : 20] Input [9]: [channel#111, id#112, spark_grouping_id#113, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] @@ -621,7 +621,7 @@ Input [1]: [d_date_sk#14] (109) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=22] Subquery:2 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt index 1671811424..d66aa0fbef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt @@ -58,7 +58,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (9) CometExchange Input [5]: [count#21, sum#22, count#23, sum#24, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometHashAggregate Input [5]: [count#21, sum#22, count#23, sum#24, count#25] @@ -109,7 +109,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (17) CometExchange Input [5]: [count#34, sum#35, count#36, sum#37, count#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (18) CometHashAggregate Input [5]: [count#34, sum#35, count#36, sum#37, count#38] @@ -160,7 +160,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (25) CometExchange Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (26) CometHashAggregate Input [5]: [count#47, sum#48, count#49, sum#50, count#51] @@ -211,7 +211,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (33) CometExchange Input [5]: [count#60, sum#61, count#62, sum#63, count#64] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (34) CometHashAggregate Input [5]: [count#60, sum#61, count#62, sum#63, count#64] @@ -262,7 +262,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (41) CometExchange Input [5]: [count#73, sum#74, count#75, sum#76, count#77] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (42) CometHashAggregate Input [5]: [count#73, sum#74, count#75, sum#76, count#77] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt index 4b396dbc38..5e011dac84 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt @@ -36,7 +36,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s (3) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (4) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] @@ -59,7 +59,7 @@ Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity (8) CometExchange Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), Exchange hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (9) CometSort Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] @@ -97,7 +97,7 @@ Input [1]: [r_reason_sk#12] (17) BroadcastExchange Input [1]: [r_reason_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (18) BroadcastHashJoin [codegen id : 2] Left keys [1]: [sr_reason_sk#8] @@ -118,7 +118,7 @@ Results [3]: [ss_customer_sk#2, sum#17, isEmpty#18] (21) Exchange Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=6] (22) HashAggregate [codegen id : 3] Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt index aafe59a77e..8c52d67797 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt @@ -63,7 +63,7 @@ Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse (4) CometExchange Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#5, 5), Exchange hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (5) CometSort Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] @@ -84,7 +84,7 @@ Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order (9) CometExchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#10, 5), Exchange hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (10) CometSort Input [2]: [ws_warehouse_sk#9, ws_order_number#10] @@ -115,7 +115,7 @@ Arguments: [wr_order_number#12], [wr_order_number#12] (16) CometExchange Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#12, 5), Exchange hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (17) CometSort Input [1]: [wr_order_number#12] @@ -150,7 +150,7 @@ Input [1]: [d_date_sk#14] (24) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] (25) BroadcastHashJoin [codegen id : 8] Left keys [1]: [ws_ship_date_sk#1] @@ -182,7 +182,7 @@ Input [1]: [ca_address_sk#16] (31) BroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] (32) BroadcastHashJoin [codegen id : 8] Left keys [1]: [ws_ship_addr_sk#2] @@ -214,7 +214,7 @@ Input [1]: [web_site_sk#18] (38) BroadcastExchange Input [1]: [web_site_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] (39) BroadcastHashJoin [codegen id : 8] Left keys [1]: [ws_web_site_sk#3] @@ -249,7 +249,7 @@ Results [3]: [sum#22, sum#23, count#25] (44) Exchange Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] (45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt index 375c7bf2f9..49d1b9a85d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt @@ -1,65 +1,68 @@ == Physical Plan == -* HashAggregate (61) -+- Exchange (60) - +- * HashAggregate (59) - +- * HashAggregate (58) - +- * HashAggregate (57) - +- * Project (56) - +- * BroadcastHashJoin Inner BuildRight (55) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- * BroadcastHashJoin Inner BuildRight (41) - : : :- * SortMergeJoin LeftSemi (35) - : : : :- * SortMergeJoin LeftSemi (18) +* HashAggregate (64) ++- Exchange (63) + +- * HashAggregate (62) + +- * HashAggregate (61) + +- * HashAggregate (60) + +- * Project (59) + +- * BroadcastHashJoin Inner BuildRight (58) + :- * Project (52) + : +- * BroadcastHashJoin Inner BuildRight (51) + : :- * Project (45) + : : +- * BroadcastHashJoin Inner BuildRight (44) + : : :- * SortMergeJoin LeftSemi (38) + : : : :- * SortMergeJoin LeftSemi (21) : : : : :- * ColumnarToRow (6) : : : : : +- CometSort (5) : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (17) - : : : : +- * SortMergeJoin Inner (16) + : : : : +- * Project (20) + : : : : +- * SortMergeJoin Inner (19) : : : : :- * ColumnarToRow (12) : : : : : +- CometSort (11) : : : : : +- CometExchange (10) : : : : : +- CometProject (9) : : : : : +- CometFilter (8) : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) - : : : : +- * ColumnarToRow (15) - : : : : +- CometSort (14) - : : : : +- ReusedExchange (13) - : : : +- * Project (34) - : : : +- * SortMergeJoin Inner (33) - : : : :- * ColumnarToRow (24) - : : : : +- CometSort (23) - : : : : +- CometExchange (22) - : : : : +- CometProject (21) - : : : : +- CometFilter (20) - : : : : +- CometScan parquet spark_catalog.default.web_returns (19) - : : : +- * Project (32) - : : : +- * SortMergeJoin Inner (31) - : : : :- * ColumnarToRow (27) - : : : : +- CometSort (26) - : : : : +- ReusedExchange (25) - : : : +- * ColumnarToRow (30) - : : : +- CometSort (29) - : : : +- ReusedExchange (28) - : : +- BroadcastExchange (40) - : : +- * ColumnarToRow (39) - : : +- CometProject (38) - : : +- CometFilter (37) - : : +- CometScan parquet spark_catalog.default.date_dim (36) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometProject (45) - : +- CometFilter (44) - : +- CometScan parquet spark_catalog.default.customer_address (43) - +- BroadcastExchange (54) - +- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.web_site (50) + : : : : +- * ColumnarToRow (18) + : : : : +- CometSort (17) + : : : : +- CometExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- * Project (37) + : : : +- * SortMergeJoin Inner (36) + : : : :- * ColumnarToRow (27) + : : : : +- CometSort (26) + : : : : +- CometExchange (25) + : : : : +- CometProject (24) + : : : : +- CometFilter (23) + : : : : +- CometScan parquet spark_catalog.default.web_returns (22) + : : : +- * Project (35) + : : : +- * SortMergeJoin Inner (34) + : : : :- * ColumnarToRow (30) + : : : : +- CometSort (29) + : : : : +- ReusedExchange (28) + : : : +- * ColumnarToRow (33) + : : : +- CometSort (32) + : : : +- ReusedExchange (31) + : : +- BroadcastExchange (43) + : : +- * ColumnarToRow (42) + : : +- CometProject (41) + : : +- CometFilter (40) + : : +- CometScan parquet spark_catalog.default.date_dim (39) + : +- BroadcastExchange (50) + : +- * ColumnarToRow (49) + : +- CometProject (48) + : +- CometFilter (47) + : +- CometScan parquet spark_catalog.default.customer_address (46) + +- BroadcastExchange (57) + +- * ColumnarToRow (56) + +- CometProject (55) + +- CometFilter (54) + +- CometScan parquet spark_catalog.default.web_site (53) (unknown) Scan parquet spark_catalog.default.web_sales @@ -79,7 +82,7 @@ Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_num (4) CometExchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#4, 5), Exchange hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (5) CometSort Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] @@ -105,7 +108,7 @@ Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_ (10) CometExchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#9, 5), Exchange hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (11) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] @@ -114,229 +117,245 @@ Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS (12) ColumnarToRow [codegen id : 2] Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -(13) ReusedExchange [Reuses operator id: 10] -Output [2]: [ws_warehouse_sk#11, ws_order_number#12] +(unknown) Scan parquet spark_catalog.default.web_sales +Output [3]: [ws_warehouse_sk#11, ws_order_number#12, ws_sold_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [ws_warehouse_sk#11, ws_order_number#12, ws_sold_date_sk#13] +Condition : (isnotnull(ws_order_number#12) AND isnotnull(ws_warehouse_sk#11)) + +(15) CometProject +Input [3]: [ws_warehouse_sk#11, ws_order_number#12, ws_sold_date_sk#13] +Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_warehouse_sk#11, ws_order_number#12] -(14) CometSort +(16) CometExchange +Input [2]: [ws_warehouse_sk#11, ws_order_number#12] +Arguments: hashpartitioning(ws_order_number#12, 5), Exchange hashpartitioning(ws_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(17) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(15) ColumnarToRow [codegen id : 3] +(18) ColumnarToRow [codegen id : 3] Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -(16) SortMergeJoin [codegen id : 4] +(19) SortMergeJoin [codegen id : 4] Left keys [1]: [ws_order_number#9] Right keys [1]: [ws_order_number#12] Join type: Inner Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(17) Project [codegen id : 4] +(20) Project [codegen id : 4] Output [1]: [ws_order_number#9] Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] -(18) SortMergeJoin [codegen id : 5] +(21) SortMergeJoin [codegen id : 5] Left keys [1]: [ws_order_number#4] Right keys [1]: [ws_order_number#9] Join type: LeftSemi Join condition: None (unknown) Scan parquet spark_catalog.default.web_returns -Output [2]: [wr_order_number#13, wr_returned_date_sk#14] +Output [2]: [wr_order_number#14, wr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number)] ReadSchema: struct -(20) CometFilter -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] -Condition : isnotnull(wr_order_number#13) +(23) CometFilter +Input [2]: [wr_order_number#14, wr_returned_date_sk#15] +Condition : isnotnull(wr_order_number#14) -(21) CometProject -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] -Arguments: [wr_order_number#13], [wr_order_number#13] +(24) CometProject +Input [2]: [wr_order_number#14, wr_returned_date_sk#15] +Arguments: [wr_order_number#14], [wr_order_number#14] -(22) CometExchange -Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(25) CometExchange +Input [1]: [wr_order_number#14] +Arguments: hashpartitioning(wr_order_number#14, 5), Exchange hashpartitioning(wr_order_number#14, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(23) CometSort -Input [1]: [wr_order_number#13] -Arguments: [wr_order_number#13], [wr_order_number#13 ASC NULLS FIRST] +(26) CometSort +Input [1]: [wr_order_number#14] +Arguments: [wr_order_number#14], [wr_order_number#14 ASC NULLS FIRST] -(24) ColumnarToRow [codegen id : 6] -Input [1]: [wr_order_number#13] +(27) ColumnarToRow [codegen id : 6] +Input [1]: [wr_order_number#14] -(25) ReusedExchange [Reuses operator id: 10] +(28) ReusedExchange [Reuses operator id: 10] Output [2]: [ws_warehouse_sk#8, ws_order_number#9] -(26) CometSort +(29) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] -(27) ColumnarToRow [codegen id : 7] +(30) ColumnarToRow [codegen id : 7] Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -(28) ReusedExchange [Reuses operator id: 10] +(31) ReusedExchange [Reuses operator id: 16] Output [2]: [ws_warehouse_sk#11, ws_order_number#12] -(29) CometSort +(32) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(30) ColumnarToRow [codegen id : 8] +(33) ColumnarToRow [codegen id : 8] Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -(31) SortMergeJoin [codegen id : 9] +(34) SortMergeJoin [codegen id : 9] Left keys [1]: [ws_order_number#9] Right keys [1]: [ws_order_number#12] Join type: Inner Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(32) Project [codegen id : 9] +(35) Project [codegen id : 9] Output [1]: [ws_order_number#9] Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] -(33) SortMergeJoin [codegen id : 10] -Left keys [1]: [wr_order_number#13] +(36) SortMergeJoin [codegen id : 10] +Left keys [1]: [wr_order_number#14] Right keys [1]: [ws_order_number#9] Join type: Inner Join condition: None -(34) Project [codegen id : 10] -Output [1]: [wr_order_number#13] -Input [2]: [wr_order_number#13, ws_order_number#9] +(37) Project [codegen id : 10] +Output [1]: [wr_order_number#14] +Input [2]: [wr_order_number#14, ws_order_number#9] -(35) SortMergeJoin [codegen id : 14] +(38) SortMergeJoin [codegen id : 14] Left keys [1]: [ws_order_number#4] -Right keys [1]: [wr_order_number#13] +Right keys [1]: [wr_order_number#14] Join type: LeftSemi Join condition: None (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_date#16] +Output [2]: [d_date_sk#16, d_date#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(37) CometFilter -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 1999-02-01)) AND (d_date#16 <= 1999-04-02)) AND isnotnull(d_date_sk#15)) +(40) CometFilter +Input [2]: [d_date_sk#16, d_date#17] +Condition : (((isnotnull(d_date#17) AND (d_date#17 >= 1999-02-01)) AND (d_date#17 <= 1999-04-02)) AND isnotnull(d_date_sk#16)) -(38) CometProject -Input [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15], [d_date_sk#15] +(41) CometProject +Input [2]: [d_date_sk#16, d_date#17] +Arguments: [d_date_sk#16], [d_date_sk#16] -(39) ColumnarToRow [codegen id : 11] -Input [1]: [d_date_sk#15] +(42) ColumnarToRow [codegen id : 11] +Input [1]: [d_date_sk#16] -(40) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(43) BroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] -(41) BroadcastHashJoin [codegen id : 14] +(44) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#15] +Right keys [1]: [d_date_sk#16] Join type: Inner Join condition: None -(42) Project [codegen id : 14] +(45) Project [codegen id : 14] Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#15] +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#16] (unknown) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] +Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] ReadSchema: struct -(44) CometFilter -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : ((isnotnull(ca_state#18) AND (ca_state#18 = IL)) AND isnotnull(ca_address_sk#17)) +(47) CometFilter +Input [2]: [ca_address_sk#18, ca_state#19] +Condition : ((isnotnull(ca_state#19) AND (ca_state#19 = IL)) AND isnotnull(ca_address_sk#18)) -(45) CometProject -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: [ca_address_sk#17], [ca_address_sk#17] +(48) CometProject +Input [2]: [ca_address_sk#18, ca_state#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] -(46) ColumnarToRow [codegen id : 12] -Input [1]: [ca_address_sk#17] +(49) ColumnarToRow [codegen id : 12] +Input [1]: [ca_address_sk#18] -(47) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(50) BroadcastExchange +Input [1]: [ca_address_sk#18] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] -(48) BroadcastHashJoin [codegen id : 14] +(51) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#17] +Right keys [1]: [ca_address_sk#18] Join type: Inner Join condition: None -(49) Project [codegen id : 14] +(52) Project [codegen id : 14] Output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#17] +Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#18] (unknown) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#19, web_company_name#20] +Output [2]: [web_site_sk#20, web_company_name#21] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] ReadSchema: struct -(51) CometFilter -Input [2]: [web_site_sk#19, web_company_name#20] -Condition : ((isnotnull(web_company_name#20) AND (web_company_name#20 = pri )) AND isnotnull(web_site_sk#19)) +(54) CometFilter +Input [2]: [web_site_sk#20, web_company_name#21] +Condition : ((isnotnull(web_company_name#21) AND (web_company_name#21 = pri )) AND isnotnull(web_site_sk#20)) -(52) CometProject -Input [2]: [web_site_sk#19, web_company_name#20] -Arguments: [web_site_sk#19], [web_site_sk#19] +(55) CometProject +Input [2]: [web_site_sk#20, web_company_name#21] +Arguments: [web_site_sk#20], [web_site_sk#20] -(53) ColumnarToRow [codegen id : 13] -Input [1]: [web_site_sk#19] +(56) ColumnarToRow [codegen id : 13] +Input [1]: [web_site_sk#20] -(54) BroadcastExchange -Input [1]: [web_site_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(57) BroadcastExchange +Input [1]: [web_site_sk#20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] -(55) BroadcastHashJoin [codegen id : 14] +(58) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#19] +Right keys [1]: [web_site_sk#20] Join type: Inner Join condition: None -(56) Project [codegen id : 14] +(59) Project [codegen id : 14] Output [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#19] +Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#20] -(57) HashAggregate [codegen id : 14] +(60) HashAggregate [codegen id : 14] Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Keys [1]: [ws_order_number#4] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23] +Results [3]: [ws_order_number#4, sum#24, sum#25] -(58) HashAggregate [codegen id : 14] -Input [3]: [ws_order_number#4, sum#23, sum#24] +(61) HashAggregate [codegen id : 14] +Input [3]: [ws_order_number#4, sum#24, sum#25] Keys [1]: [ws_order_number#4] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23] +Results [3]: [ws_order_number#4, sum#24, sum#25] -(59) HashAggregate [codegen id : 14] -Input [3]: [ws_order_number#4, sum#23, sum#24] +(62) HashAggregate [codegen id : 14] +Input [3]: [ws_order_number#4, sum#24, sum#25] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [sum#23, sum#24, count#26] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23, count(ws_order_number#4)#26] +Results [3]: [sum#24, sum#25, count#27] -(60) Exchange -Input [3]: [sum#23, sum#24, count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(63) Exchange +Input [3]: [sum#24, sum#25, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] -(61) HashAggregate [codegen id : 15] -Input [3]: [sum#23, sum#24, count#26] +(64) HashAggregate [codegen id : 15] +Input [3]: [sum#24, sum#25, count#27] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [count(ws_order_number#4)#25 AS order count #27, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#21,17,2) AS total shipping cost #28, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#22,17,2) AS total net profit #29] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23, count(ws_order_number#4)#26] +Results [3]: [count(ws_order_number#4)#26 AS order count #28, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#22,17,2) AS total shipping cost #29, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#23,17,2) AS total net profit #30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt index 2ad651cb64..49ecbf500d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt @@ -43,7 +43,10 @@ WholeStageCodegen (15) ColumnarToRow InputAdapter CometSort [ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometExchange [ws_order_number] #4 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_order_number,ws_warehouse_sk] + CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] InputAdapter WholeStageCodegen (10) Project [wr_order_number] @@ -53,7 +56,7 @@ WholeStageCodegen (15) ColumnarToRow InputAdapter CometSort [wr_order_number] - CometExchange [wr_order_number] #4 + CometExchange [wr_order_number] #5 CometProject [wr_order_number] CometFilter [wr_order_number] CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] @@ -72,9 +75,9 @@ WholeStageCodegen (15) ColumnarToRow InputAdapter CometSort [ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + ReusedExchange [ws_warehouse_sk,ws_order_number] #4 InputAdapter - BroadcastExchange #5 + BroadcastExchange #6 WholeStageCodegen (11) ColumnarToRow InputAdapter @@ -82,7 +85,7 @@ WholeStageCodegen (15) CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #6 + BroadcastExchange #7 WholeStageCodegen (12) ColumnarToRow InputAdapter @@ -90,7 +93,7 @@ WholeStageCodegen (15) CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #7 + BroadcastExchange #8 WholeStageCodegen (13) ColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt index 9bbf5e9847..6101b5e09e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt @@ -65,7 +65,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s (4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -88,7 +88,7 @@ Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8 (9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), Exchange hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -126,7 +126,7 @@ Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (18) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#3] @@ -154,7 +154,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (24) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] (25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -182,7 +182,7 @@ Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name (30) BroadcastExchange Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] (31) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_customer_sk#2] @@ -210,7 +210,7 @@ Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] (36) BroadcastExchange Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=8] (37) BroadcastHashJoin [codegen id : 5] Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] @@ -231,7 +231,7 @@ Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_ (40) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=9] (41) HashAggregate [codegen id : 6] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] @@ -249,7 +249,7 @@ Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty# (43) Exchange Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] (44) HashAggregate [codegen id : 7] Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] @@ -264,7 +264,7 @@ Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subqu (46) Exchange Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=11] (47) Sort [codegen id : 8] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] @@ -358,7 +358,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (61) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] (62) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -405,7 +405,7 @@ Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_ (71) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=13] (72) HashAggregate [codegen id : 6] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] @@ -423,7 +423,7 @@ Results [2]: [sum#46, count#47] (74) Exchange Input [2]: [sum#46, count#47] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=14] (75) HashAggregate [codegen id : 7] Input [2]: [sum#46, count#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt index fcaa84890b..78abb49dee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt @@ -156,7 +156,7 @@ Functions [1]: [partial_avg(UnscaledValue(i_current_price#13))] (25) CometExchange Input [3]: [i_category#14, sum#15, count#16] -Arguments: hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#14, 5), Exchange hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometHashAggregate Input [3]: [i_category#14, sum#15, count#16] @@ -172,7 +172,7 @@ Input [2]: [avg(i_current_price)#17, i_category#14] (29) BroadcastExchange Input [2]: [avg(i_current_price)#17, i_category#14] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] (30) BroadcastHashJoin [codegen id : 5] Left keys [1]: [i_category#12] @@ -186,7 +186,7 @@ Input [5]: [i_item_sk#10, i_current_price#11, i_category#12, avg(i_current_price (32) BroadcastExchange Input [1]: [i_item_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (33) BroadcastHashJoin [codegen id : 6] Left keys [1]: [ss_item_sk#5] @@ -207,7 +207,7 @@ Results [2]: [ca_state#2, count#19] (36) Exchange Input [2]: [ca_state#2, count#19] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] (37) HashAggregate [codegen id : 7] Input [2]: [ca_state#2, count#19] @@ -254,7 +254,7 @@ Input [1]: [d_date_sk#9] (44) BroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] Subquery:2 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#24, [id=#25] * ColumnarToRow (51) @@ -288,7 +288,7 @@ Functions: [] (49) CometExchange Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(d_month_seq#26, 5), Exchange hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (50) CometHashAggregate Input [1]: [d_month_seq#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt index 1429e39daa..ce84534b1a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) +* Sort (192) ++- Exchange (191) + +- * Project (190) + +- * SortMergeJoin Inner (189) :- * Sort (108) : +- Exchange (107) : +- * HashAggregate (106) @@ -111,43 +111,43 @@ : +- CometProject (100) : +- CometFilter (99) : +- CometScan parquet spark_catalog.default.item (98) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * ColumnarToRow (126) - : : : : : : : : : : : : : : : +- CometProject (125) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (124) + +- * Sort (188) + +- Exchange (187) + +- * HashAggregate (186) + +- * HashAggregate (185) + +- * Project (184) + +- * BroadcastHashJoin Inner BuildRight (183) + :- * Project (181) + : +- * BroadcastHashJoin Inner BuildRight (180) + : :- * Project (178) + : : +- * BroadcastHashJoin Inner BuildRight (177) + : : :- * Project (175) + : : : +- * BroadcastHashJoin Inner BuildRight (174) + : : : :- * Project (172) + : : : : +- * BroadcastHashJoin Inner BuildRight (171) + : : : : :- * Project (169) + : : : : : +- * BroadcastHashJoin Inner BuildRight (168) + : : : : : :- * Project (166) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (165) + : : : : : : :- * Project (163) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (162) + : : : : : : : :- * Project (160) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (159) + : : : : : : : : :- * Project (157) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (156) + : : : : : : : : : :- * Project (154) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) + : : : : : : : : : : :- * Project (151) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) + : : : : : : : : : : : :- * Project (148) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) + : : : : : : : : : : : : :- * Project (145) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) + : : : : : : : : : : : : : :- * Project (142) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) + : : : : : : : : : : : : : : :- * ColumnarToRow (139) + : : : : : : : : : : : : : : : +- CometProject (138) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (137) : : : : : : : : : : : : : : : :- CometSort (118) : : : : : : : : : : : : : : : : +- CometExchange (117) : : : : : : : : : : : : : : : : +- CometProject (116) @@ -158,26 +158,39 @@ : : : : : : : : : : : : : : : : +- CometProject (114) : : : : : : : : : : : : : : : : +- CometFilter (113) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (112) - : : : : : : : : : : : : : : : +- CometSort (123) - : : : : : : : : : : : : : : : +- CometProject (122) - : : : : : : : : : : : : : : : +- CometFilter (121) - : : : : : : : : : : : : : : : +- CometHashAggregate (120) - : : : : : : : : : : : : : : : +- ReusedExchange (119) - : : : : : : : : : : : : : : +- ReusedExchange (127) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) + : : : : : : : : : : : : : : : +- CometSort (136) + : : : : : : : : : : : : : : : +- CometProject (135) + : : : : : : : : : : : : : : : +- CometFilter (134) + : : : : : : : : : : : : : : : +- CometHashAggregate (133) + : : : : : : : : : : : : : : : +- CometExchange (132) + : : : : : : : : : : : : : : : +- CometHashAggregate (131) + : : : : : : : : : : : : : : : +- CometProject (130) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (129) + : : : : : : : : : : : : : : : :- CometSort (123) + : : : : : : : : : : : : : : : : +- CometExchange (122) + : : : : : : : : : : : : : : : : +- CometProject (121) + : : : : : : : : : : : : : : : : +- CometFilter (120) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (119) + : : : : : : : : : : : : : : : +- CometSort (128) + : : : : : : : : : : : : : : : +- CometExchange (127) + : : : : : : : : : : : : : : : +- CometProject (126) + : : : : : : : : : : : : : : : +- CometFilter (125) + : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (124) + : : : : : : : : : : : : : : +- ReusedExchange (140) + : : : : : : : : : : : : : +- ReusedExchange (143) + : : : : : : : : : : : : +- ReusedExchange (146) + : : : : : : : : : : : +- ReusedExchange (149) + : : : : : : : : : : +- ReusedExchange (152) + : : : : : : : : : +- ReusedExchange (155) + : : : : : : : : +- ReusedExchange (158) + : : : : : : : +- ReusedExchange (161) + : : : : : : +- ReusedExchange (164) + : : : : : +- ReusedExchange (167) + : : : : +- ReusedExchange (170) + : : : +- ReusedExchange (173) + : : +- ReusedExchange (176) + : +- ReusedExchange (179) + +- ReusedExchange (182) (unknown) Scan parquet spark_catalog.default.store_sales @@ -222,7 +235,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), Exchange hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -245,7 +258,7 @@ Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_s (14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), Exchange hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -268,7 +281,7 @@ Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_ (19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), Exchange hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -290,7 +303,7 @@ Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum((( (24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, 5), Exchange hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -321,7 +334,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (31) ColumnarToRow [codegen id : 16] Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(32) ReusedExchange [Reuses operator id: 183] +(32) ReusedExchange [Reuses operator id: 196] Output [2]: [d_date_sk#32, d_year#33] (33) BroadcastHashJoin [codegen id : 16] @@ -350,7 +363,7 @@ Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] (38) BroadcastExchange Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (39) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_store_sk#6] @@ -378,7 +391,7 @@ Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_cu (44) BroadcastExchange Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] (45) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_customer_sk#2] @@ -406,7 +419,7 @@ Input [2]: [d_date_sk#43, d_year#44] (50) BroadcastExchange Input [2]: [d_date_sk#43, d_year#44] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] (51) BroadcastHashJoin [codegen id : 16] Left keys [1]: [c_first_sales_date_sk#42] @@ -447,7 +460,7 @@ Input [2]: [cd_demo_sk#47, cd_marital_status#48] (59) BroadcastExchange Input [2]: [cd_demo_sk#47, cd_marital_status#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] (60) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_cdemo_sk#3] @@ -488,7 +501,7 @@ Input [1]: [p_promo_sk#51] (68) BroadcastExchange Input [1]: [p_promo_sk#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] (69) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_promo_sk#7] @@ -516,7 +529,7 @@ Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] (74) BroadcastExchange Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] (75) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_hdemo_sk#4] @@ -557,7 +570,7 @@ Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59 (83) BroadcastExchange Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] (84) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_addr_sk#5] @@ -598,7 +611,7 @@ Input [1]: [ib_income_band_sk#66] (92) BroadcastExchange Input [1]: [ib_income_band_sk#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] (93) BroadcastHashJoin [codegen id : 16] Left keys [1]: [hd_income_band_sk#53] @@ -643,7 +656,7 @@ Input [2]: [i_item_sk#68, i_product_name#71] (102) BroadcastExchange Input [2]: [i_item_sk#68, i_product_name#71] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] (103) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_item_sk#1] @@ -671,7 +684,7 @@ Results [17]: [i_product_name#71 AS product_name#84, i_item_sk#68 AS item_sk#85, (107) Exchange Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] -Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=18] (108) Sort [codegen id : 17] Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] @@ -719,286 +732,347 @@ Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104 (117) CometExchange Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -Arguments: hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +Arguments: hashpartitioning(ss_item_sk#101, 5), Exchange hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=19], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=20] (118) CometSort Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101 ASC NULLS FIRST] -(119) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] +(unknown) Scan parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct + +(120) CometFilter +Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] +Condition : (isnotnull(cs_item_sk#117) AND isnotnull(cs_order_number#118)) + +(121) CometProject +Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] +Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] + +(122) CometExchange +Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] +Arguments: hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), Exchange hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] + +(123) CometSort +Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] +Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117 ASC NULLS FIRST, cs_order_number#118 ASC NULLS FIRST] + +(unknown) Scan parquet spark_catalog.default.catalog_returns +Output [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(125) CometFilter +Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] +Condition : (isnotnull(cr_item_sk#121) AND isnotnull(cr_order_number#122)) + +(126) CometProject +Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] +Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] + +(127) CometExchange +Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), Exchange hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), ENSURE_REQUIREMENTS, [plan_id=23], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=24] + +(128) CometSort +Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121 ASC NULLS FIRST, cr_order_number#122 ASC NULLS FIRST] + +(129) CometSortMergeJoin +Left output [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] +Right output [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: [cs_item_sk#117, cs_order_number#118], [cr_item_sk#121, cr_order_number#122], Inner -(120) CometHashAggregate -Input [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] +(130) CometProject +Input [8]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +Arguments: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] + +(131) CometHashAggregate +Input [5]: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] Keys [1]: [cs_item_sk#117] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#121)), sum(((cr_refunded_cash#122 + cr_reversed_charge#123) + cr_store_credit#124))] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#119)), partial_sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] + +(132) CometExchange +Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(cs_item_sk#117, 5), Exchange hashpartitioning(cs_item_sk#117, 5), ENSURE_REQUIREMENTS, [plan_id=25], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=26] -(121) CometFilter +(133) CometHashAggregate +Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] +Keys [1]: [cs_item_sk#117] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#119)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] + +(134) CometFilter Input [3]: [cs_item_sk#117, sale#30, refund#31] Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) -(122) CometProject +(135) CometProject Input [3]: [cs_item_sk#117, sale#30, refund#31] Arguments: [cs_item_sk#117], [cs_item_sk#117] -(123) CometSort +(136) CometSort Input [1]: [cs_item_sk#117] Arguments: [cs_item_sk#117], [cs_item_sk#117 ASC NULLS FIRST] -(124) CometSortMergeJoin +(137) CometSortMergeJoin Left output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Right output [1]: [cs_item_sk#117] Arguments: [ss_item_sk#101], [cs_item_sk#117], Inner -(125) CometProject +(138) CometProject Input [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, cs_item_sk#117] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(126) ColumnarToRow [codegen id : 33] +(139) ColumnarToRow [codegen id : 33] Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(127) ReusedExchange [Reuses operator id: 187] -Output [2]: [d_date_sk#125, d_year#126] +(140) ReusedExchange [Reuses operator id: 200] +Output [2]: [d_date_sk#130, d_year#131] -(128) BroadcastHashJoin [codegen id : 33] +(141) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_sold_date_sk#112] -Right keys [1]: [d_date_sk#125] +Right keys [1]: [d_date_sk#130] Join type: Inner Join condition: None -(129) Project [codegen id : 33] -Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126] -Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#125, d_year#126] +(142) Project [codegen id : 33] +Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131] +Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#130, d_year#131] -(130) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#127, s_store_name#128, s_zip#129] +(143) ReusedExchange [Reuses operator id: 38] +Output [3]: [s_store_sk#132, s_store_name#133, s_zip#134] -(131) BroadcastHashJoin [codegen id : 33] +(144) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_store_sk#106] -Right keys [1]: [s_store_sk#127] +Right keys [1]: [s_store_sk#132] Join type: Inner Join condition: None -(132) Project [codegen id : 33] -Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129] -Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_sk#127, s_store_name#128, s_zip#129] +(145) Project [codegen id : 33] +Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134] +Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_sk#132, s_store_name#133, s_zip#134] -(133) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] +(146) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] -(134) BroadcastHashJoin [codegen id : 33] +(147) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_customer_sk#102] -Right keys [1]: [c_customer_sk#130] +Right keys [1]: [c_customer_sk#135] Join type: Inner Join condition: None -(135) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] -Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] +(148) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] +Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] -(136) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#136, d_year#137] +(149) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#141, d_year#142] -(137) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_sales_date_sk#135] -Right keys [1]: [d_date_sk#136] +(150) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_sales_date_sk#140] +Right keys [1]: [d_date_sk#141] Join type: Inner Join condition: None -(138) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135, d_date_sk#136, d_year#137] +(151) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140, d_date_sk#141, d_year#142] -(139) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#138, d_year#139] +(152) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#143, d_year#144] -(140) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_shipto_date_sk#134] -Right keys [1]: [d_date_sk#138] +(153) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_shipto_date_sk#139] +Right keys [1]: [d_date_sk#143] Join type: Inner Join condition: None -(141) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137, d_date_sk#138, d_year#139] +(154) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142, d_date_sk#143, d_year#144] -(142) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#140, cd_marital_status#141] +(155) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] -(143) BroadcastHashJoin [codegen id : 33] +(156) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_cdemo_sk#103] -Right keys [1]: [cd_demo_sk#140] +Right keys [1]: [cd_demo_sk#145] Join type: Inner Join condition: None -(144) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_demo_sk#140, cd_marital_status#141] +(157) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_demo_sk#145, cd_marital_status#146] -(145) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#142, cd_marital_status#143] +(158) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#147, cd_marital_status#148] -(146) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_cdemo_sk#131] -Right keys [1]: [cd_demo_sk#142] +(159) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_cdemo_sk#136] +Right keys [1]: [cd_demo_sk#147] Join type: Inner -Join condition: NOT (cd_marital_status#141 = cd_marital_status#143) +Join condition: NOT (cd_marital_status#146 = cd_marital_status#148) -(147) Project [codegen id : 33] -Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] -Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141, cd_demo_sk#142, cd_marital_status#143] +(160) Project [codegen id : 33] +Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] +Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146, cd_demo_sk#147, cd_marital_status#148] -(148) ReusedExchange [Reuses operator id: 68] -Output [1]: [p_promo_sk#144] +(161) ReusedExchange [Reuses operator id: 68] +Output [1]: [p_promo_sk#149] -(149) BroadcastHashJoin [codegen id : 33] +(162) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_promo_sk#107] -Right keys [1]: [p_promo_sk#144] +Right keys [1]: [p_promo_sk#149] Join type: Inner Join condition: None -(150) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, p_promo_sk#144] +(163) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, p_promo_sk#149] -(151) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#145, hd_income_band_sk#146] +(164) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] -(152) BroadcastHashJoin [codegen id : 33] +(165) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_hdemo_sk#104] -Right keys [1]: [hd_demo_sk#145] +Right keys [1]: [hd_demo_sk#150] Join type: Inner Join condition: None -(153) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_demo_sk#145, hd_income_band_sk#146] +(166) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_demo_sk#150, hd_income_band_sk#151] -(154) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#147, hd_income_band_sk#148] +(167) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] -(155) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_hdemo_sk#132] -Right keys [1]: [hd_demo_sk#147] +(168) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_hdemo_sk#137] +Right keys [1]: [hd_demo_sk#152] Join type: Inner Join condition: None -(156) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148] -Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_demo_sk#147, hd_income_band_sk#148] +(169) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153] +Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_demo_sk#152, hd_income_band_sk#153] -(157) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] +(170) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(158) BroadcastHashJoin [codegen id : 33] +(171) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_addr_sk#105] -Right keys [1]: [ca_address_sk#149] +Right keys [1]: [ca_address_sk#154] Join type: Inner Join condition: None -(159) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] -Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] +(172) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(160) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(173) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -(161) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_addr_sk#133] -Right keys [1]: [ca_address_sk#154] +(174) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_addr_sk#138] +Right keys [1]: [ca_address_sk#159] Join type: Inner Join condition: None -(162) Project [codegen id : 33] -Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(175) Project [codegen id : 33] +Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -(163) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#159] +(176) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#164] -(164) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#146] -Right keys [1]: [ib_income_band_sk#159] +(177) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#164] Join type: Inner Join condition: None -(165) Project [codegen id : 33] -Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#159] +(178) Project [codegen id : 33] +Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#164] -(166) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#160] +(179) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#165] -(167) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#148] -Right keys [1]: [ib_income_band_sk#160] +(180) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#153] +Right keys [1]: [ib_income_band_sk#165] Join type: Inner Join condition: None -(168) Project [codegen id : 33] -Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#160] +(181) Project [codegen id : 33] +Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#165] -(169) ReusedExchange [Reuses operator id: 102] -Output [2]: [i_item_sk#161, i_product_name#162] +(182) ReusedExchange [Reuses operator id: 102] +Output [2]: [i_item_sk#166, i_product_name#167] -(170) BroadcastHashJoin [codegen id : 33] +(183) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_item_sk#101] -Right keys [1]: [i_item_sk#161] +Right keys [1]: [i_item_sk#166] Join type: Inner Join condition: None -(171) Project [codegen id : 33] -Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] +(184) Project [codegen id : 33] +Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] -(172) HashAggregate [codegen id : 33] -Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] -Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] +(185) HashAggregate [codegen id : 33] +Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] +Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#109)), partial_sum(UnscaledValue(ss_list_price#110)), partial_sum(UnscaledValue(ss_coupon_amt#111))] -Aggregate Attributes [4]: [count#72, sum#163, sum#164, sum#165] -Results [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] +Aggregate Attributes [4]: [count#72, sum#168, sum#169, sum#170] +Results [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] -(173) HashAggregate [codegen id : 33] -Input [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] -Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] +(186) HashAggregate [codegen id : 33] +Input [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] +Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#109)), sum(UnscaledValue(ss_list_price#110)), sum(UnscaledValue(ss_coupon_amt#111))] Aggregate Attributes [4]: [count(1)#80, sum(UnscaledValue(ss_wholesale_cost#109))#81, sum(UnscaledValue(ss_list_price#110))#82, sum(UnscaledValue(ss_coupon_amt#111))#83] -Results [8]: [i_item_sk#161 AS item_sk#169, s_store_name#128 AS store_name#170, s_zip#129 AS store_zip#171, d_year#126 AS syear#172, count(1)#80 AS cnt#173, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#174, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#175, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#176] +Results [8]: [i_item_sk#166 AS item_sk#174, s_store_name#133 AS store_name#175, s_zip#134 AS store_zip#176, d_year#131 AS syear#177, count(1)#80 AS cnt#178, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#179, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#180, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#181] -(174) Exchange -Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(187) Exchange +Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] +Arguments: hashpartitioning(item_sk#174, store_name#175, store_zip#176, 5), ENSURE_REQUIREMENTS, [plan_id=27] -(175) Sort [codegen id : 34] -Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -Arguments: [item_sk#169 ASC NULLS FIRST, store_name#170 ASC NULLS FIRST, store_zip#171 ASC NULLS FIRST], false, 0 +(188) Sort [codegen id : 34] +Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] +Arguments: [item_sk#174 ASC NULLS FIRST, store_name#175 ASC NULLS FIRST, store_zip#176 ASC NULLS FIRST], false, 0 -(176) SortMergeJoin [codegen id : 35] +(189) SortMergeJoin [codegen id : 35] Left keys [3]: [item_sk#85, store_name#86, store_zip#87] -Right keys [3]: [item_sk#169, store_name#170, store_zip#171] +Right keys [3]: [item_sk#174, store_name#175, store_zip#176] Join type: Inner -Join condition: (cnt#173 <= cnt#97) +Join condition: (cnt#178 <= cnt#97) -(177) Project [codegen id : 35] -Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] +(190) Project [codegen id : 35] +Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] +Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] -(178) Exchange -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#174 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=17] +(191) Exchange +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] +Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#179 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=28] -(179) Sort [codegen id : 36] -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#174 ASC NULLS FIRST], true, 0 +(192) Sort [codegen id : 36] +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] +Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#179 ASC NULLS FIRST], true, 0 ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (183) -+- * ColumnarToRow (182) - +- CometFilter (181) - +- CometScan parquet spark_catalog.default.date_dim (180) +BroadcastExchange (196) ++- * ColumnarToRow (195) + +- CometFilter (194) + +- CometScan parquet spark_catalog.default.date_dim (193) (unknown) Scan parquet spark_catalog.default.date_dim @@ -1008,40 +1082,40 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(181) CometFilter +(194) CometFilter Input [2]: [d_date_sk#32, d_year#33] Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) -(182) ColumnarToRow [codegen id : 1] +(195) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#32, d_year#33] -(183) BroadcastExchange +(196) BroadcastExchange Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=29] Subquery:2 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#112 IN dynamicpruning#113 -BroadcastExchange (187) -+- * ColumnarToRow (186) - +- CometFilter (185) - +- CometScan parquet spark_catalog.default.date_dim (184) +BroadcastExchange (200) ++- * ColumnarToRow (199) + +- CometFilter (198) + +- CometScan parquet spark_catalog.default.date_dim (197) (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#125, d_year#126] +Output [2]: [d_date_sk#130, d_year#131] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(185) CometFilter -Input [2]: [d_date_sk#125, d_year#126] -Condition : ((isnotnull(d_year#126) AND (d_year#126 = 2000)) AND isnotnull(d_date_sk#125)) +(198) CometFilter +Input [2]: [d_date_sk#130, d_year#131] +Condition : ((isnotnull(d_year#131) AND (d_year#131 = 2000)) AND isnotnull(d_date_sk#130)) -(186) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#125, d_year#126] +(199) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#130, d_year#131] -(187) BroadcastExchange -Input [2]: [d_date_sk#125, d_year#126] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] +(200) BroadcastExchange +Input [2]: [d_date_sk#130, d_year#131] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt index e2accd7418..3e9778636a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt @@ -221,7 +221,20 @@ WholeStageCodegen (36) CometProject [cs_item_sk] CometFilter [sale,refund] CometHashAggregate [cs_item_sk,sum,sum,isEmpty] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 + CometExchange [cs_item_sk] #22 + CometHashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number] + CometExchange [cs_item_sk,cs_order_number] #23 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #24 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter ReusedExchange [d_date_sk,d_year] #21 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt index 46b9e51d5e..0da99bb7dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt @@ -357,7 +357,7 @@ Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number# (62) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), Exchange hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=10], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (63) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] @@ -385,7 +385,7 @@ Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] (68) Exchange Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=12] (69) HashAggregate [codegen id : 14] Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] @@ -428,6 +428,6 @@ Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] (75) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index 3922f7efe6..1bf96469f0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -TakeOrderedAndProject (132) -+- * Project (131) - +- * SortMergeJoin Inner (130) +TakeOrderedAndProject (141) ++- * Project (140) + +- * SortMergeJoin Inner (139) :- * Sort (71) : +- Exchange (70) : +- * Filter (69) @@ -73,18 +73,18 @@ TakeOrderedAndProject (132) : +- CometProject (56) : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.web_returns (54) - +- * Sort (129) - +- Exchange (128) - +- * Filter (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- * HashAggregate (123) - +- Exchange (122) - +- * HashAggregate (121) - +- Union (120) - :- * Project (87) - : +- * SortMergeJoin LeftOuter (86) + +- * Sort (138) + +- Exchange (137) + +- * Filter (136) + +- * HashAggregate (135) + +- Exchange (134) + +- * HashAggregate (133) + +- * HashAggregate (132) + +- Exchange (131) + +- * HashAggregate (130) + +- Union (129) + :- * Project (90) + : +- * SortMergeJoin LeftOuter (89) : :- * Sort (82) : : +- Exchange (81) : : +- * Project (80) @@ -96,41 +96,50 @@ TakeOrderedAndProject (132) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) : : : +- ReusedExchange (75) : : +- ReusedExchange (78) - : +- * ColumnarToRow (85) - : +- CometSort (84) - : +- ReusedExchange (83) - :- * Project (103) - : +- * SortMergeJoin LeftOuter (102) - : :- * Sort (98) - : : +- Exchange (97) - : : +- * Project (96) - : : +- * BroadcastHashJoin Inner BuildRight (95) - : : :- * Project (93) - : : : +- * BroadcastHashJoin Inner BuildRight (92) - : : : :- * ColumnarToRow (90) - : : : : +- CometFilter (89) - : : : : +- CometScan parquet spark_catalog.default.store_sales (88) - : : : +- ReusedExchange (91) - : : +- ReusedExchange (94) - : +- * ColumnarToRow (101) - : +- CometSort (100) - : +- ReusedExchange (99) - +- * Project (119) - +- * SortMergeJoin LeftOuter (118) - :- * Sort (114) - : +- Exchange (113) - : +- * Project (112) - : +- * BroadcastHashJoin Inner BuildRight (111) - : :- * Project (109) - : : +- * BroadcastHashJoin Inner BuildRight (108) - : : :- * ColumnarToRow (106) - : : : +- CometFilter (105) - : : : +- CometScan parquet spark_catalog.default.web_sales (104) - : : +- ReusedExchange (107) - : +- ReusedExchange (110) - +- * ColumnarToRow (117) - +- CometSort (116) - +- ReusedExchange (115) + : +- * ColumnarToRow (88) + : +- CometSort (87) + : +- CometExchange (86) + : +- CometProject (85) + : +- CometFilter (84) + : +- CometScan parquet spark_catalog.default.catalog_returns (83) + :- * Project (109) + : +- * SortMergeJoin LeftOuter (108) + : :- * Sort (101) + : : +- Exchange (100) + : : +- * Project (99) + : : +- * BroadcastHashJoin Inner BuildRight (98) + : : :- * Project (96) + : : : +- * BroadcastHashJoin Inner BuildRight (95) + : : : :- * ColumnarToRow (93) + : : : : +- CometFilter (92) + : : : : +- CometScan parquet spark_catalog.default.store_sales (91) + : : : +- ReusedExchange (94) + : : +- ReusedExchange (97) + : +- * ColumnarToRow (107) + : +- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometFilter (103) + : +- CometScan parquet spark_catalog.default.store_returns (102) + +- * Project (128) + +- * SortMergeJoin LeftOuter (127) + :- * Sort (120) + : +- Exchange (119) + : +- * Project (118) + : +- * BroadcastHashJoin Inner BuildRight (117) + : :- * Project (115) + : : +- * BroadcastHashJoin Inner BuildRight (114) + : : :- * ColumnarToRow (112) + : : : +- CometFilter (111) + : : : +- CometScan parquet spark_catalog.default.web_sales (110) + : : +- ReusedExchange (113) + : +- ReusedExchange (116) + +- * ColumnarToRow (126) + +- CometSort (125) + +- CometExchange (124) + +- CometProject (123) + +- CometFilter (122) + +- CometScan parquet spark_catalog.default.web_returns (121) (unknown) Scan parquet spark_catalog.default.catalog_sales @@ -180,7 +189,7 @@ Join condition: None Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -(11) ReusedExchange [Reuses operator id: 136] +(11) ReusedExchange [Reuses operator id: 145] Output [2]: [d_date_sk#13, d_year#14] (12) BroadcastHashJoin [codegen id : 3] @@ -218,7 +227,7 @@ Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_ (19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), Exchange hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -265,7 +274,7 @@ Join condition: None Output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -(30) ReusedExchange [Reuses operator id: 136] +(30) ReusedExchange [Reuses operator id: 145] Output [2]: [d_date_sk#33, d_year#34] (31) BroadcastHashJoin [codegen id : 9] @@ -280,7 +289,7 @@ Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_pr (33) Exchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=5] (34) Sort [codegen id : 10] Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -303,7 +312,7 @@ Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return (38) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), Exchange hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (39) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -350,7 +359,7 @@ Join condition: None Output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -(49) ReusedExchange [Reuses operator id: 136] +(49) ReusedExchange [Reuses operator id: 145] Output [2]: [d_date_sk#53, d_year#54] (50) BroadcastHashJoin [codegen id : 15] @@ -365,7 +374,7 @@ Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_pri (52) Exchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] (53) Sort [codegen id : 16] Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -388,7 +397,7 @@ Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_ (57) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), Exchange hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (58) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -418,7 +427,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (64) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=11] (65) HashAggregate [codegen id : 20] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -436,7 +445,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (67) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=12] (68) HashAggregate [codegen id : 21] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] @@ -451,7 +460,7 @@ Condition : isnotnull(sales_cnt#68) (70) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=13] (71) Sort [codegen id : 22] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] @@ -485,7 +494,7 @@ Join condition: None Output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Input [10]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(78) ReusedExchange [Reuses operator id: 140] +(78) ReusedExchange [Reuses operator id: 149] Output [2]: [d_date_sk#81, d_year#82] (79) BroadcastHashJoin [codegen id : 25] @@ -500,241 +509,289 @@ Input [11]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_pri (81) Exchange Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=14] (82) Sort [codegen id : 26] Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Arguments: [cs_order_number#71 ASC NULLS FIRST, cs_item_sk#70 ASC NULLS FIRST], false, 0 -(83) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] +(unknown) Scan parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(84) CometFilter +Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] +Condition : (isnotnull(cr_order_number#84) AND isnotnull(cr_item_sk#83)) -(84) CometSort +(85) CometProject +Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] +Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] + +(86) CometExchange +Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] +Arguments: hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), Exchange hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] + +(87) CometSort Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_order_number#84 ASC NULLS FIRST, cr_item_sk#83 ASC NULLS FIRST] -(85) ColumnarToRow [codegen id : 27] +(88) ColumnarToRow [codegen id : 27] Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(86) SortMergeJoin [codegen id : 28] +(89) SortMergeJoin [codegen id : 28] Left keys [2]: [cs_order_number#71, cs_item_sk#70] Right keys [2]: [cr_order_number#84, cr_item_sk#83] Join type: LeftOuter Join condition: None -(87) Project [codegen id : 28] +(90) Project [codegen id : 28] Output [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, (cs_quantity#72 - coalesce(cr_return_quantity#85, 0)) AS sales_cnt#20, (cs_ext_sales_price#73 - coalesce(cr_return_amount#86, 0.00)) AS sales_amt#21] Input [13]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82, cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] (unknown) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +Output [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#91), dynamicpruningexpression(ss_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ss_sold_date_sk#92), dynamicpruningexpression(ss_sold_date_sk#92 IN dynamicpruning#93)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(89) CometFilter -Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -Condition : isnotnull(ss_item_sk#87) +(92) CometFilter +Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] +Condition : isnotnull(ss_item_sk#88) -(90) ColumnarToRow [codegen id : 31] -Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +(93) ColumnarToRow [codegen id : 31] +Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] -(91) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +(94) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] -(92) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_item_sk#87] -Right keys [1]: [i_item_sk#93] +(95) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_item_sk#88] +Right keys [1]: [i_item_sk#94] Join type: Inner Join condition: None -(93) Project [codegen id : 31] -Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Input [10]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +(96) Project [codegen id : 31] +Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] +Input [10]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] -(94) ReusedExchange [Reuses operator id: 140] -Output [2]: [d_date_sk#98, d_year#99] +(97) ReusedExchange [Reuses operator id: 149] +Output [2]: [d_date_sk#99, d_year#100] -(95) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_sold_date_sk#91] -Right keys [1]: [d_date_sk#98] +(98) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_sold_date_sk#92] +Right keys [1]: [d_date_sk#99] Join type: Inner Join condition: None -(96) Project [codegen id : 31] -Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_date_sk#98, d_year#99] +(99) Project [codegen id : 31] +Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] +Input [11]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_date_sk#99, d_year#100] -(97) Exchange -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(100) Exchange +Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] +Arguments: hashpartitioning(ss_ticket_number#89, ss_item_sk#88, 5), ENSURE_REQUIREMENTS, [plan_id=17] -(98) Sort [codegen id : 32] -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: [ss_ticket_number#88 ASC NULLS FIRST, ss_item_sk#87 ASC NULLS FIRST], false, 0 +(101) Sort [codegen id : 32] +Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] +Arguments: [ss_ticket_number#89 ASC NULLS FIRST, ss_item_sk#88 ASC NULLS FIRST], false, 0 -(99) ReusedExchange [Reuses operator id: 38] -Output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +(unknown) Scan parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(103) CometFilter +Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] +Condition : (isnotnull(sr_ticket_number#102) AND isnotnull(sr_item_sk#101)) -(100) CometSort -Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103], [sr_ticket_number#101 ASC NULLS FIRST, sr_item_sk#100 ASC NULLS FIRST] +(104) CometProject +Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] +Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] -(101) ColumnarToRow [codegen id : 33] -Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +(105) CometExchange +Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +Arguments: hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), Exchange hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=18], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=19] -(102) SortMergeJoin [codegen id : 34] -Left keys [2]: [ss_ticket_number#88, ss_item_sk#87] -Right keys [2]: [sr_ticket_number#101, sr_item_sk#100] +(106) CometSort +Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_ticket_number#102 ASC NULLS FIRST, sr_item_sk#101 ASC NULLS FIRST] + +(107) ColumnarToRow [codegen id : 33] +Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] + +(108) SortMergeJoin [codegen id : 34] +Left keys [2]: [ss_ticket_number#89, ss_item_sk#88] +Right keys [2]: [sr_ticket_number#102, sr_item_sk#101] Join type: LeftOuter Join condition: None -(103) Project [codegen id : 34] -Output [7]: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, (ss_quantity#89 - coalesce(sr_return_quantity#102, 0)) AS sales_cnt#40, (ss_ext_sales_price#90 - coalesce(sr_return_amt#103, 0.00)) AS sales_amt#41] -Input [13]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99, sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +(109) Project [codegen id : 34] +Output [7]: [d_year#100, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, (ss_quantity#90 - coalesce(sr_return_quantity#103, 0)) AS sales_cnt#40, (ss_ext_sales_price#91 - coalesce(sr_return_amt#104, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100, sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] (unknown) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] +Output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#108), dynamicpruningexpression(ws_sold_date_sk#108 IN dynamicpruning#109)] +PartitionFilters: [isnotnull(ws_sold_date_sk#110), dynamicpruningexpression(ws_sold_date_sk#110 IN dynamicpruning#111)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(105) CometFilter -Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] -Condition : isnotnull(ws_item_sk#104) +(111) CometFilter +Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +Condition : isnotnull(ws_item_sk#106) -(106) ColumnarToRow [codegen id : 37] -Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] +(112) ColumnarToRow [codegen id : 37] +Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -(107) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] +(113) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -(108) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_item_sk#104] -Right keys [1]: [i_item_sk#110] +(114) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_item_sk#106] +Right keys [1]: [i_item_sk#112] Join type: Inner Join condition: None -(109) Project [codegen id : 37] -Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] -Input [10]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] +(115) Project [codegen id : 37] +Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +Input [10]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -(110) ReusedExchange [Reuses operator id: 140] -Output [2]: [d_date_sk#115, d_year#116] +(116) ReusedExchange [Reuses operator id: 149] +Output [2]: [d_date_sk#117, d_year#118] -(111) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_sold_date_sk#108] -Right keys [1]: [d_date_sk#115] +(117) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_sold_date_sk#110] +Right keys [1]: [d_date_sk#117] Join type: Inner Join condition: None -(112) Project [codegen id : 37] -Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Input [11]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_date_sk#115, d_year#116] +(118) Project [codegen id : 37] +Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Input [11]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_date_sk#117, d_year#118] + +(119) Exchange +Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Arguments: hashpartitioning(ws_order_number#107, ws_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(120) Sort [codegen id : 38] +Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Arguments: [ws_order_number#107 ASC NULLS FIRST, ws_item_sk#106 ASC NULLS FIRST], false, 0 + +(unknown) Scan parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct -(113) Exchange -Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(122) CometFilter +Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] +Condition : (isnotnull(wr_order_number#120) AND isnotnull(wr_item_sk#119)) -(114) Sort [codegen id : 38] -Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Arguments: [ws_order_number#105 ASC NULLS FIRST, ws_item_sk#104 ASC NULLS FIRST], false, 0 +(123) CometProject +Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] +Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -(115) ReusedExchange [Reuses operator id: 57] -Output [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +(124) CometExchange +Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +Arguments: hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), Exchange hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] -(116) CometSort -Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -Arguments: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120], [wr_order_number#118 ASC NULLS FIRST, wr_item_sk#117 ASC NULLS FIRST] +(125) CometSort +Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_order_number#120 ASC NULLS FIRST, wr_item_sk#119 ASC NULLS FIRST] -(117) ColumnarToRow [codegen id : 39] -Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +(126) ColumnarToRow [codegen id : 39] +Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -(118) SortMergeJoin [codegen id : 40] -Left keys [2]: [ws_order_number#105, ws_item_sk#104] -Right keys [2]: [wr_order_number#118, wr_item_sk#117] +(127) SortMergeJoin [codegen id : 40] +Left keys [2]: [ws_order_number#107, ws_item_sk#106] +Right keys [2]: [wr_order_number#120, wr_item_sk#119] Join type: LeftOuter Join condition: None -(119) Project [codegen id : 40] -Output [7]: [d_year#116, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, (ws_quantity#106 - coalesce(wr_return_quantity#119, 0)) AS sales_cnt#60, (ws_ext_sales_price#107 - coalesce(wr_return_amt#120, 0.00)) AS sales_amt#61] -Input [13]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116, wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +(128) Project [codegen id : 40] +Output [7]: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, (ws_quantity#108 - coalesce(wr_return_quantity#121, 0)) AS sales_cnt#60, (ws_ext_sales_price#109 - coalesce(wr_return_amt#122, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118, wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -(120) Union +(129) Union -(121) HashAggregate [codegen id : 41] +(130) HashAggregate [codegen id : 41] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(122) Exchange +(131) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=23] -(123) HashAggregate [codegen id : 42] +(132) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(124) HashAggregate [codegen id : 42] +(133) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -Aggregate Attributes [2]: [sum#62, sum#121] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] +Aggregate Attributes [2]: [sum#62, sum#124] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] -(125) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(134) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=24] -(126) HashAggregate [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] +(135) HashAggregate [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Aggregate Attributes [2]: [sum(sales_cnt#20)#66, sum(UnscaledValue(sales_amt#21))#67] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#123, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#124] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#126, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#127] -(127) Filter [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -Condition : isnotnull(sales_cnt#123) +(136) Filter [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +Condition : isnotnull(sales_cnt#126) -(128) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(137) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=25] -(129) Sort [codegen id : 44] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +(138) Sort [codegen id : 44] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] Arguments: [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST], false, 0 -(130) SortMergeJoin [codegen id : 45] +(139) SortMergeJoin [codegen id : 45] Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Right keys [4]: [i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Join type: Inner -Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#123 as decimal(17,2))) < 0.90000000000000000000) +Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#126 as decimal(17,2))) < 0.90000000000000000000) -(131) Project [codegen id : 45] -Output [10]: [d_year#82 AS prev_year#125, d_year#14 AS year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#123 AS prev_yr_cnt#127, sales_cnt#68 AS curr_yr_cnt#128, (sales_cnt#68 - sales_cnt#123) AS sales_cnt_diff#129, (sales_amt#69 - sales_amt#124) AS sales_amt_diff#130] -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +(140) Project [codegen id : 45] +Output [10]: [d_year#82 AS prev_year#128, d_year#14 AS year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#126 AS prev_yr_cnt#130, sales_cnt#68 AS curr_yr_cnt#131, (sales_cnt#68 - sales_cnt#126) AS sales_cnt_diff#132, (sales_amt#69 - sales_amt#127) AS sales_amt_diff#133] +Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] -(132) TakeOrderedAndProject -Input [10]: [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] -Arguments: 100, [sales_cnt_diff#129 ASC NULLS FIRST, sales_amt_diff#130 ASC NULLS FIRST], [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] +(141) TakeOrderedAndProject +Input [10]: [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] +Arguments: 100, [sales_cnt_diff#132 ASC NULLS FIRST, sales_amt_diff#133 ASC NULLS FIRST], [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (136) -+- * ColumnarToRow (135) - +- CometFilter (134) - +- CometScan parquet spark_catalog.default.date_dim (133) +BroadcastExchange (145) ++- * ColumnarToRow (144) + +- CometFilter (143) + +- CometScan parquet spark_catalog.default.date_dim (142) (unknown) Scan parquet spark_catalog.default.date_dim @@ -744,26 +801,26 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(134) CometFilter +(143) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(135) ColumnarToRow [codegen id : 1] +(144) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -(136) BroadcastExchange +(145) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=26] Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 -BroadcastExchange (140) -+- * ColumnarToRow (139) - +- CometFilter (138) - +- CometScan parquet spark_catalog.default.date_dim (137) +BroadcastExchange (149) ++- * ColumnarToRow (148) + +- CometFilter (147) + +- CometScan parquet spark_catalog.default.date_dim (146) (unknown) Scan parquet spark_catalog.default.date_dim @@ -773,19 +830,19 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(138) CometFilter +(147) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(139) ColumnarToRow [codegen id : 1] +(148) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -(140) BroadcastExchange +(149) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=27] -Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#75 +Subquery:5 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#92 IN dynamicpruning#75 -Subquery:6 Hosting operator id = 104 Hosting Expression = ws_sold_date_sk#108 IN dynamicpruning#75 +Subquery:6 Hosting operator id = 110 Hosting Expression = ws_sold_date_sk#110 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt index 25dd0f9468..1dc6c29cba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt @@ -178,7 +178,10 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [cr_order_number,cr_item_sk] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometExchange [cr_order_number,cr_item_sk] #17 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_order_number,cr_item_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] WholeStageCodegen (34) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] @@ -186,7 +189,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i WholeStageCodegen (32) Sort [ss_ticket_number,ss_item_sk] InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 + Exchange [ss_ticket_number,ss_item_sk] #18 WholeStageCodegen (31) Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] @@ -206,7 +209,10 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [sr_ticket_number,sr_item_sk] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometExchange [sr_ticket_number,sr_item_sk] #19 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_ticket_number,sr_item_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] WholeStageCodegen (40) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] @@ -214,7 +220,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i WholeStageCodegen (38) Sort [ws_order_number,ws_item_sk] InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 + Exchange [ws_order_number,ws_item_sk] #20 WholeStageCodegen (37) Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] @@ -234,4 +240,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [wr_order_number,wr_item_sk] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 + CometExchange [wr_order_number,wr_item_sk] #21 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_order_number,wr_item_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt index cfb9ed7d0e..3d7e571162 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt @@ -82,7 +82,7 @@ Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) (3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -105,7 +105,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number# (8) CometExchange Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), Exchange hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (9) CometSort Input [2]: [sr_item_sk#9, sr_ticket_number#10] @@ -149,7 +149,7 @@ Results [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] (18) Exchange Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=5] (19) HashAggregate [codegen id : 3] Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] @@ -176,7 +176,7 @@ Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) (23) CometExchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), Exchange hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (24) CometSort Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] @@ -199,7 +199,7 @@ Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number# (28) CometExchange Input [2]: [wr_item_sk#35, wr_order_number#36] -Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), Exchange hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=8], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (29) CometSort Input [2]: [wr_item_sk#35, wr_order_number#36] @@ -243,7 +243,7 @@ Results [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, (38) Exchange Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=10] (39) HashAggregate [codegen id : 6] Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] @@ -284,7 +284,7 @@ Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) (46) CometExchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), Exchange hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (47) CometSort Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] @@ -307,7 +307,7 @@ Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number# (51) CometExchange Input [2]: [cr_item_sk#62, cr_order_number#63] -Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), Exchange hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, [plan_id=13], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (52) CometSort Input [2]: [cr_item_sk#62, cr_order_number#63] @@ -351,7 +351,7 @@ Results [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, (61) Exchange Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=15] (62) HashAggregate [codegen id : 10] Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] @@ -407,7 +407,7 @@ Input [2]: [d_date_sk#12, d_year#13] (71) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] Subquery:2 Hosting operator id = 21 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt index dfd081b442..334eab7c7b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -TakeOrderedAndProject (117) -+- * HashAggregate (116) - +- Exchange (115) - +- * HashAggregate (114) - +- Union (113) +TakeOrderedAndProject (265) ++- * HashAggregate (264) + +- Exchange (263) + +- * HashAggregate (262) + +- Union (261) :- * HashAggregate (102) : +- Exchange (101) : +- * HashAggregate (100) @@ -106,16 +106,164 @@ TakeOrderedAndProject (117) : : : +- CometScan parquet spark_catalog.default.web_site (84) : : +- ReusedExchange (90) : +- ReusedExchange (93) - :- * HashAggregate (107) - : +- Exchange (106) - : +- * HashAggregate (105) - : +- * HashAggregate (104) - : +- ReusedExchange (103) - +- * HashAggregate (112) - +- Exchange (111) - +- * HashAggregate (110) - +- * HashAggregate (109) - +- ReusedExchange (108) + :- * HashAggregate (181) + : +- Exchange (180) + : +- * HashAggregate (179) + : +- * HashAggregate (178) + : +- Exchange (177) + : +- * HashAggregate (176) + : +- Union (175) + : :- * HashAggregate (126) + : : +- Exchange (125) + : : +- * HashAggregate (124) + : : +- * Project (123) + : : +- * BroadcastHashJoin Inner BuildRight (122) + : : :- * Project (120) + : : : +- * BroadcastHashJoin Inner BuildRight (119) + : : : :- * Project (117) + : : : : +- * BroadcastHashJoin Inner BuildRight (116) + : : : : :- * Project (114) + : : : : : +- * BroadcastHashJoin Inner BuildRight (113) + : : : : : :- * ColumnarToRow (111) + : : : : : : +- CometProject (110) + : : : : : : +- CometSortMergeJoin (109) + : : : : : : :- CometSort (106) + : : : : : : : +- CometExchange (105) + : : : : : : : +- CometFilter (104) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (103) + : : : : : : +- CometSort (108) + : : : : : : +- ReusedExchange (107) + : : : : : +- ReusedExchange (112) + : : : : +- ReusedExchange (115) + : : : +- ReusedExchange (118) + : : +- ReusedExchange (121) + : :- * HashAggregate (150) + : : +- Exchange (149) + : : +- * HashAggregate (148) + : : +- * Project (147) + : : +- * BroadcastHashJoin Inner BuildRight (146) + : : :- * Project (144) + : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : :- * Project (141) + : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : :- * Project (138) + : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : :- * ColumnarToRow (135) + : : : : : : +- CometProject (134) + : : : : : : +- CometSortMergeJoin (133) + : : : : : : :- CometSort (130) + : : : : : : : +- CometExchange (129) + : : : : : : : +- CometFilter (128) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (127) + : : : : : : +- CometSort (132) + : : : : : : +- ReusedExchange (131) + : : : : : +- ReusedExchange (136) + : : : : +- ReusedExchange (139) + : : : +- ReusedExchange (142) + : : +- ReusedExchange (145) + : +- * HashAggregate (174) + : +- Exchange (173) + : +- * HashAggregate (172) + : +- * Project (171) + : +- * BroadcastHashJoin Inner BuildRight (170) + : :- * Project (168) + : : +- * BroadcastHashJoin Inner BuildRight (167) + : : :- * Project (165) + : : : +- * BroadcastHashJoin Inner BuildRight (164) + : : : :- * Project (162) + : : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : : :- * ColumnarToRow (159) + : : : : : +- CometProject (158) + : : : : : +- CometSortMergeJoin (157) + : : : : : :- CometSort (154) + : : : : : : +- CometExchange (153) + : : : : : : +- CometFilter (152) + : : : : : : +- CometScan parquet spark_catalog.default.web_sales (151) + : : : : : +- CometSort (156) + : : : : : +- ReusedExchange (155) + : : : : +- ReusedExchange (160) + : : : +- ReusedExchange (163) + : : +- ReusedExchange (166) + : +- ReusedExchange (169) + +- * HashAggregate (260) + +- Exchange (259) + +- * HashAggregate (258) + +- * HashAggregate (257) + +- Exchange (256) + +- * HashAggregate (255) + +- Union (254) + :- * HashAggregate (205) + : +- Exchange (204) + : +- * HashAggregate (203) + : +- * Project (202) + : +- * BroadcastHashJoin Inner BuildRight (201) + : :- * Project (199) + : : +- * BroadcastHashJoin Inner BuildRight (198) + : : :- * Project (196) + : : : +- * BroadcastHashJoin Inner BuildRight (195) + : : : :- * Project (193) + : : : : +- * BroadcastHashJoin Inner BuildRight (192) + : : : : :- * ColumnarToRow (190) + : : : : : +- CometProject (189) + : : : : : +- CometSortMergeJoin (188) + : : : : : :- CometSort (185) + : : : : : : +- CometExchange (184) + : : : : : : +- CometFilter (183) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (182) + : : : : : +- CometSort (187) + : : : : : +- ReusedExchange (186) + : : : : +- ReusedExchange (191) + : : : +- ReusedExchange (194) + : : +- ReusedExchange (197) + : +- ReusedExchange (200) + :- * HashAggregate (229) + : +- Exchange (228) + : +- * HashAggregate (227) + : +- * Project (226) + : +- * BroadcastHashJoin Inner BuildRight (225) + : :- * Project (223) + : : +- * BroadcastHashJoin Inner BuildRight (222) + : : :- * Project (220) + : : : +- * BroadcastHashJoin Inner BuildRight (219) + : : : :- * Project (217) + : : : : +- * BroadcastHashJoin Inner BuildRight (216) + : : : : :- * ColumnarToRow (214) + : : : : : +- CometProject (213) + : : : : : +- CometSortMergeJoin (212) + : : : : : :- CometSort (209) + : : : : : : +- CometExchange (208) + : : : : : : +- CometFilter (207) + : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (206) + : : : : : +- CometSort (211) + : : : : : +- ReusedExchange (210) + : : : : +- ReusedExchange (215) + : : : +- ReusedExchange (218) + : : +- ReusedExchange (221) + : +- ReusedExchange (224) + +- * HashAggregate (253) + +- Exchange (252) + +- * HashAggregate (251) + +- * Project (250) + +- * BroadcastHashJoin Inner BuildRight (249) + :- * Project (247) + : +- * BroadcastHashJoin Inner BuildRight (246) + : :- * Project (244) + : : +- * BroadcastHashJoin Inner BuildRight (243) + : : :- * Project (241) + : : : +- * BroadcastHashJoin Inner BuildRight (240) + : : : :- * ColumnarToRow (238) + : : : : +- CometProject (237) + : : : : +- CometSortMergeJoin (236) + : : : : :- CometSort (233) + : : : : : +- CometExchange (232) + : : : : : +- CometFilter (231) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (230) + : : : : +- CometSort (235) + : : : : +- ReusedExchange (234) + : : : +- ReusedExchange (239) + : : +- ReusedExchange (242) + : +- ReusedExchange (245) + +- ReusedExchange (248) (unknown) Scan parquet spark_catalog.default.store_sales @@ -132,7 +280,7 @@ Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnul (3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -155,7 +303,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] (8) CometExchange Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), Exchange hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (9) CometSort Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] @@ -173,7 +321,7 @@ Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss (12) ColumnarToRow [codegen id : 5] Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -(13) ReusedExchange [Reuses operator id: 122] +(13) ReusedExchange [Reuses operator id: 270] Output [1]: [d_date_sk#14] (14) BroadcastHashJoin [codegen id : 5] @@ -202,7 +350,7 @@ Input [2]: [s_store_sk#15, s_store_id#16] (19) BroadcastExchange Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (20) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#2] @@ -234,7 +382,7 @@ Input [1]: [i_item_sk#17] (26) BroadcastExchange Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (27) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -266,7 +414,7 @@ Input [1]: [p_promo_sk#19] (33) BroadcastExchange Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_promo_sk#3] @@ -287,7 +435,7 @@ Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] (37) Exchange Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=8] (38) HashAggregate [codegen id : 6] Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] @@ -310,7 +458,7 @@ Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND (41) CometExchange Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (42) CometSort Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] @@ -333,7 +481,7 @@ Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss# (46) CometExchange Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), Exchange hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (47) CometSort Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] @@ -351,7 +499,7 @@ Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_p (50) ColumnarToRow [codegen id : 11] Input [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] -(51) ReusedExchange [Reuses operator id: 122] +(51) ReusedExchange [Reuses operator id: 270] Output [1]: [d_date_sk#52] (52) BroadcastHashJoin [codegen id : 11] @@ -380,7 +528,7 @@ Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] (57) BroadcastExchange Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] (58) BroadcastHashJoin [codegen id : 11] Left keys [1]: [cs_catalog_page_sk#39] @@ -427,7 +575,7 @@ Results [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty (67) Exchange Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] -Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=14] (68) HashAggregate [codegen id : 12] Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] @@ -450,7 +598,7 @@ Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isn (71) CometExchange Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] (72) CometSort Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] @@ -473,7 +621,7 @@ Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] (76) CometExchange Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), Exchange hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, [plan_id=17], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] (77) CometSort Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] @@ -491,7 +639,7 @@ Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price (80) ColumnarToRow [codegen id : 17] Input [8]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] -(81) ReusedExchange [Reuses operator id: 122] +(81) ReusedExchange [Reuses operator id: 270] Output [1]: [d_date_sk#88] (82) BroadcastHashJoin [codegen id : 17] @@ -520,7 +668,7 @@ Input [2]: [web_site_sk#89, web_site_id#90] (87) BroadcastExchange Input [2]: [web_site_sk#89, web_site_id#90] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] (88) BroadcastHashJoin [codegen id : 17] Left keys [1]: [ws_web_site_sk#76] @@ -567,7 +715,7 @@ Results [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (97) Exchange Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=20] (98) HashAggregate [codegen id : 18] Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] @@ -587,7 +735,7 @@ Results [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum (101) Exchange Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=21] (102) HashAggregate [codegen id : 20] Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] @@ -596,120 +744,806 @@ Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#123, sum(returns#37)#124, sum(profit#38)#125] Results [5]: [channel#34, id#35, cast(sum(sales#36)#123 as decimal(37,2)) AS sales#126, cast(sum(returns#37)#124 as decimal(38,2)) AS returns#127, cast(sum(profit#38)#125 as decimal(38,2)) AS profit#128] -(103) ReusedExchange [Reuses operator id: 101] -Output [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] +(unknown) Scan parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#129)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct + +(104) CometFilter +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) + +(105) CometExchange +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=22], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=23] + +(106) CometSort +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] + +(107) ReusedExchange [Reuses operator id: 8] +Output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] + +(108) CometSort +Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] + +(109) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter + +(110) CometProject +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] + +(111) ColumnarToRow [codegen id : 25] +Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] + +(112) ReusedExchange [Reuses operator id: 270] +Output [1]: [d_date_sk#14] + +(113) BroadcastHashJoin [codegen id : 25] +Left keys [1]: [ss_sold_date_sk#7] +Right keys [1]: [d_date_sk#14] +Join type: Inner +Join condition: None + +(114) Project [codegen id : 25] +Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] + +(115) ReusedExchange [Reuses operator id: 19] +Output [2]: [s_store_sk#15, s_store_id#16] + +(116) BroadcastHashJoin [codegen id : 25] +Left keys [1]: [ss_store_sk#2] +Right keys [1]: [s_store_sk#15] +Join type: Inner +Join condition: None + +(117) Project [codegen id : 25] +Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#15, s_store_id#16] + +(118) ReusedExchange [Reuses operator id: 26] +Output [1]: [i_item_sk#17] + +(119) BroadcastHashJoin [codegen id : 25] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [i_item_sk#17] +Join type: Inner +Join condition: None + +(120) Project [codegen id : 25] +Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, i_item_sk#17] + +(121) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#19] + +(122) BroadcastHashJoin [codegen id : 25] +Left keys [1]: [ss_promo_sk#3] +Right keys [1]: [p_promo_sk#19] +Join type: Inner +Join condition: None + +(123) Project [codegen id : 25] +Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, p_promo_sk#19] + +(124) HashAggregate [codegen id : 25] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Keys [1]: [s_store_id#16] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Aggregate Attributes [5]: [sum#130, sum#131, isEmpty#132, sum#133, isEmpty#134] +Results [6]: [s_store_id#16, sum#135, sum#136, isEmpty#137, sum#138, isEmpty#139] + +(125) Exchange +Input [6]: [s_store_id#16, sum#135, sum#136, isEmpty#137, sum#138, isEmpty#139] +Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=24] + +(126) HashAggregate [codegen id : 26] +Input [6]: [s_store_id#16, sum#135, sum#136, isEmpty#137, sum#138, isEmpty#139] +Keys [1]: [s_store_id#16] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#16) AS id#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33 AS profit#38] + +(unknown) Scan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_sold_date_sk#45 IN dynamicpruning#140)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct + +(128) CometFilter +Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) + +(129) CometExchange +Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=25], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=26] + +(130) CometSort +Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45], [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST] + +(131) ReusedExchange [Reuses operator id: 46] +Output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] + +(132) CometSort +Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50], [cr_item_sk#47 ASC NULLS FIRST, cr_order_number#48 ASC NULLS FIRST] + +(133) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Right output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +Arguments: [cs_item_sk#40, cs_order_number#42], [cr_item_sk#47, cr_order_number#48], LeftOuter + +(134) CometProject +Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] + +(135) ColumnarToRow [codegen id : 31] +Input [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] + +(136) ReusedExchange [Reuses operator id: 270] +Output [1]: [d_date_sk#52] + +(137) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [cs_sold_date_sk#45] +Right keys [1]: [d_date_sk#52] +Join type: Inner +Join condition: None + +(138) Project [codegen id : 31] +Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50] +Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50, d_date_sk#52] + +(139) ReusedExchange [Reuses operator id: 57] +Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] + +(140) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [cs_catalog_page_sk#39] +Right keys [1]: [cp_catalog_page_sk#53] +Join type: Inner +Join condition: None + +(141) Project [codegen id : 31] +Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_sk#53, cp_catalog_page_id#54] + +(142) ReusedExchange [Reuses operator id: 26] +Output [1]: [i_item_sk#55] + +(143) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [cs_item_sk#40] +Right keys [1]: [i_item_sk#55] +Join type: Inner +Join condition: None + +(144) Project [codegen id : 31] +Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, i_item_sk#55] + +(145) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#56] + +(146) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [cs_promo_sk#41] +Right keys [1]: [p_promo_sk#56] +Join type: Inner +Join condition: None + +(147) Project [codegen id : 31] +Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, p_promo_sk#56] + +(148) HashAggregate [codegen id : 31] +Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Keys [1]: [cp_catalog_page_id#54] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] +Aggregate Attributes [5]: [sum#141, sum#142, isEmpty#143, sum#144, isEmpty#145] +Results [6]: [cp_catalog_page_id#54, sum#146, sum#147, isEmpty#148, sum#149, isEmpty#150] + +(149) Exchange +Input [6]: [cp_catalog_page_id#54, sum#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=27] + +(150) HashAggregate [codegen id : 32] +Input [6]: [cp_catalog_page_id#54, sum#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Keys [1]: [cp_catalog_page_id#54] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#67, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69] +Results [5]: [catalog channel AS channel#70, concat(catalog_page, cp_catalog_page_id#54) AS id#71, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#67,17,2) AS sales#72, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68 AS returns#73, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69 AS profit#74] + +(unknown) Scan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#151)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(152) CometFilter +Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) + +(153) CometExchange +Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=28], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=29] + +(154) CometSort +Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81], [ws_item_sk#75 ASC NULLS FIRST, ws_order_number#78 ASC NULLS FIRST] + +(155) ReusedExchange [Reuses operator id: 76] +Output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] + +(156) CometSort +Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86], [wr_item_sk#83 ASC NULLS FIRST, wr_order_number#84 ASC NULLS FIRST] + +(157) CometSortMergeJoin +Left output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Right output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +Arguments: [ws_item_sk#75, ws_order_number#78], [wr_item_sk#83, wr_order_number#84], LeftOuter + +(158) CometProject +Input [11]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86], [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] + +(159) ColumnarToRow [codegen id : 37] +Input [8]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] + +(160) ReusedExchange [Reuses operator id: 270] +Output [1]: [d_date_sk#88] + +(161) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_sold_date_sk#81] +Right keys [1]: [d_date_sk#88] +Join type: Inner +Join condition: None + +(162) Project [codegen id : 37] +Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86] +Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86, d_date_sk#88] + +(163) ReusedExchange [Reuses operator id: 87] +Output [2]: [web_site_sk#89, web_site_id#90] -(104) HashAggregate [codegen id : 40] +(164) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_web_site_sk#76] +Right keys [1]: [web_site_sk#89] +Join type: Inner +Join condition: None + +(165) Project [codegen id : 37] +Output [7]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_sk#89, web_site_id#90] + +(166) ReusedExchange [Reuses operator id: 26] +Output [1]: [i_item_sk#91] + +(167) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_item_sk#75] +Right keys [1]: [i_item_sk#91] +Join type: Inner +Join condition: None + +(168) Project [codegen id : 37] +Output [6]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Input [8]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, i_item_sk#91] + +(169) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#92] + +(170) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_promo_sk#77] +Right keys [1]: [p_promo_sk#92] +Join type: Inner +Join condition: None + +(171) Project [codegen id : 37] +Output [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Input [7]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, p_promo_sk#92] + +(172) HashAggregate [codegen id : 37] +Input [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Keys [1]: [web_site_id#90] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#79)), partial_sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] +Aggregate Attributes [5]: [sum#152, sum#153, isEmpty#154, sum#155, isEmpty#156] +Results [6]: [web_site_id#90, sum#157, sum#158, isEmpty#159, sum#160, isEmpty#161] + +(173) Exchange +Input [6]: [web_site_id#90, sum#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=30] + +(174) HashAggregate [codegen id : 38] +Input [6]: [web_site_id#90, sum#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Keys [1]: [web_site_id#90] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#79)), sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#79))#103, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105] +Results [5]: [web channel AS channel#106, concat(web_site, web_site_id#90) AS id#107, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#79))#103,17,2) AS sales#108, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104 AS returns#109, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105 AS profit#110] + +(175) Union + +(176) HashAggregate [codegen id : 39] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116] +Results [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] + +(177) Exchange +Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=31] + +(178) HashAggregate [codegen id : 40] Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#123, sum(returns#37)#124, sum(profit#38)#125] -Results [4]: [channel#34, sum(sales#36)#123 AS sales#129, sum(returns#37)#124 AS returns#130, sum(profit#38)#125 AS profit#131] +Results [4]: [channel#34, sum(sales#36)#123 AS sales#162, sum(returns#37)#124 AS returns#163, sum(profit#38)#125 AS profit#164] -(105) HashAggregate [codegen id : 40] -Input [4]: [channel#34, sales#129, returns#130, profit#131] +(179) HashAggregate [codegen id : 40] +Input [4]: [channel#34, sales#162, returns#163, profit#164] Keys [1]: [channel#34] -Functions [3]: [partial_sum(sales#129), partial_sum(returns#130), partial_sum(profit#131)] -Aggregate Attributes [6]: [sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] -Results [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +Functions [3]: [partial_sum(sales#162), partial_sum(returns#163), partial_sum(profit#164)] +Aggregate Attributes [6]: [sum#165, isEmpty#166, sum#167, isEmpty#168, sum#169, isEmpty#170] +Results [7]: [channel#34, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] -(106) Exchange -Input [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(180) Exchange +Input [7]: [channel#34, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=32] -(107) HashAggregate [codegen id : 41] -Input [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +(181) HashAggregate [codegen id : 41] +Input [7]: [channel#34, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] Keys [1]: [channel#34] -Functions [3]: [sum(sales#129), sum(returns#130), sum(profit#131)] -Aggregate Attributes [3]: [sum(sales#129)#144, sum(returns#130)#145, sum(profit#131)#146] -Results [5]: [channel#34, null AS id#147, sum(sales#129)#144 AS sales#148, sum(returns#130)#145 AS returns#149, sum(profit#131)#146 AS profit#150] +Functions [3]: [sum(sales#162), sum(returns#163), sum(profit#164)] +Aggregate Attributes [3]: [sum(sales#162)#177, sum(returns#163)#178, sum(profit#164)#179] +Results [5]: [channel#34, null AS id#180, sum(sales#162)#177 AS sales#181, sum(returns#163)#178 AS returns#182, sum(profit#164)#179 AS profit#183] + +(unknown) Scan parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#184)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct + +(183) CometFilter +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) + +(184) CometExchange +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=33], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=34] + +(185) CometSort +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] + +(186) ReusedExchange [Reuses operator id: 8] +Output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] + +(187) CometSort +Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] + +(188) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter + +(189) CometProject +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] + +(190) ColumnarToRow [codegen id : 46] +Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] + +(191) ReusedExchange [Reuses operator id: 270] +Output [1]: [d_date_sk#14] + +(192) BroadcastHashJoin [codegen id : 46] +Left keys [1]: [ss_sold_date_sk#7] +Right keys [1]: [d_date_sk#14] +Join type: Inner +Join condition: None + +(193) Project [codegen id : 46] +Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] + +(194) ReusedExchange [Reuses operator id: 19] +Output [2]: [s_store_sk#15, s_store_id#16] + +(195) BroadcastHashJoin [codegen id : 46] +Left keys [1]: [ss_store_sk#2] +Right keys [1]: [s_store_sk#15] +Join type: Inner +Join condition: None + +(196) Project [codegen id : 46] +Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#15, s_store_id#16] + +(197) ReusedExchange [Reuses operator id: 26] +Output [1]: [i_item_sk#17] + +(198) BroadcastHashJoin [codegen id : 46] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [i_item_sk#17] +Join type: Inner +Join condition: None + +(199) Project [codegen id : 46] +Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, i_item_sk#17] + +(200) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#19] + +(201) BroadcastHashJoin [codegen id : 46] +Left keys [1]: [ss_promo_sk#3] +Right keys [1]: [p_promo_sk#19] +Join type: Inner +Join condition: None + +(202) Project [codegen id : 46] +Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, p_promo_sk#19] + +(203) HashAggregate [codegen id : 46] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] +Keys [1]: [s_store_id#16] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Aggregate Attributes [5]: [sum#185, sum#186, isEmpty#187, sum#188, isEmpty#189] +Results [6]: [s_store_id#16, sum#190, sum#191, isEmpty#192, sum#193, isEmpty#194] + +(204) Exchange +Input [6]: [s_store_id#16, sum#190, sum#191, isEmpty#192, sum#193, isEmpty#194] +Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=35] + +(205) HashAggregate [codegen id : 47] +Input [6]: [s_store_id#16, sum#190, sum#191, isEmpty#192, sum#193, isEmpty#194] +Keys [1]: [s_store_id#16] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#16) AS id#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33 AS profit#38] + +(unknown) Scan parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_sold_date_sk#45 IN dynamicpruning#195)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct + +(207) CometFilter +Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) + +(208) CometExchange +Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=36], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=37] + +(209) CometSort +Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45], [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST] + +(210) ReusedExchange [Reuses operator id: 46] +Output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] + +(211) CometSort +Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50], [cr_item_sk#47 ASC NULLS FIRST, cr_order_number#48 ASC NULLS FIRST] + +(212) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +Right output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +Arguments: [cs_item_sk#40, cs_order_number#42], [cr_item_sk#47, cr_order_number#48], LeftOuter + +(213) CometProject +Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] +Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] + +(214) ColumnarToRow [codegen id : 52] +Input [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] + +(215) ReusedExchange [Reuses operator id: 270] +Output [1]: [d_date_sk#52] + +(216) BroadcastHashJoin [codegen id : 52] +Left keys [1]: [cs_sold_date_sk#45] +Right keys [1]: [d_date_sk#52] +Join type: Inner +Join condition: None + +(217) Project [codegen id : 52] +Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50] +Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50, d_date_sk#52] + +(218) ReusedExchange [Reuses operator id: 57] +Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] + +(219) BroadcastHashJoin [codegen id : 52] +Left keys [1]: [cs_catalog_page_sk#39] +Right keys [1]: [cp_catalog_page_sk#53] +Join type: Inner +Join condition: None + +(220) Project [codegen id : 52] +Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_sk#53, cp_catalog_page_id#54] -(108) ReusedExchange [Reuses operator id: 101] -Output [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] +(221) ReusedExchange [Reuses operator id: 26] +Output [1]: [i_item_sk#55] + +(222) BroadcastHashJoin [codegen id : 52] +Left keys [1]: [cs_item_sk#40] +Right keys [1]: [i_item_sk#55] +Join type: Inner +Join condition: None + +(223) Project [codegen id : 52] +Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, i_item_sk#55] + +(224) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#56] + +(225) BroadcastHashJoin [codegen id : 52] +Left keys [1]: [cs_promo_sk#41] +Right keys [1]: [p_promo_sk#56] +Join type: Inner +Join condition: None + +(226) Project [codegen id : 52] +Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, p_promo_sk#56] + +(227) HashAggregate [codegen id : 52] +Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] +Keys [1]: [cp_catalog_page_id#54] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] +Aggregate Attributes [5]: [sum#196, sum#197, isEmpty#198, sum#199, isEmpty#200] +Results [6]: [cp_catalog_page_id#54, sum#201, sum#202, isEmpty#203, sum#204, isEmpty#205] + +(228) Exchange +Input [6]: [cp_catalog_page_id#54, sum#201, sum#202, isEmpty#203, sum#204, isEmpty#205] +Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=38] + +(229) HashAggregate [codegen id : 53] +Input [6]: [cp_catalog_page_id#54, sum#201, sum#202, isEmpty#203, sum#204, isEmpty#205] +Keys [1]: [cp_catalog_page_id#54] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#67, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69] +Results [5]: [catalog channel AS channel#70, concat(catalog_page, cp_catalog_page_id#54) AS id#71, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#67,17,2) AS sales#72, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68 AS returns#73, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69 AS profit#74] + +(unknown) Scan parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#206)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(231) CometFilter +Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) + +(232) CometExchange +Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=39], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=40] + +(233) CometSort +Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81], [ws_item_sk#75 ASC NULLS FIRST, ws_order_number#78 ASC NULLS FIRST] + +(234) ReusedExchange [Reuses operator id: 76] +Output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] + +(235) CometSort +Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86], [wr_item_sk#83 ASC NULLS FIRST, wr_order_number#84 ASC NULLS FIRST] + +(236) CometSortMergeJoin +Left output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Right output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +Arguments: [ws_item_sk#75, ws_order_number#78], [wr_item_sk#83, wr_order_number#84], LeftOuter + +(237) CometProject +Input [11]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] +Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86], [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] + +(238) ColumnarToRow [codegen id : 58] +Input [8]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] + +(239) ReusedExchange [Reuses operator id: 270] +Output [1]: [d_date_sk#88] + +(240) BroadcastHashJoin [codegen id : 58] +Left keys [1]: [ws_sold_date_sk#81] +Right keys [1]: [d_date_sk#88] +Join type: Inner +Join condition: None + +(241) Project [codegen id : 58] +Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86] +Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86, d_date_sk#88] + +(242) ReusedExchange [Reuses operator id: 87] +Output [2]: [web_site_sk#89, web_site_id#90] + +(243) BroadcastHashJoin [codegen id : 58] +Left keys [1]: [ws_web_site_sk#76] +Right keys [1]: [web_site_sk#89] +Join type: Inner +Join condition: None + +(244) Project [codegen id : 58] +Output [7]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_sk#89, web_site_id#90] -(109) HashAggregate [codegen id : 61] +(245) ReusedExchange [Reuses operator id: 26] +Output [1]: [i_item_sk#91] + +(246) BroadcastHashJoin [codegen id : 58] +Left keys [1]: [ws_item_sk#75] +Right keys [1]: [i_item_sk#91] +Join type: Inner +Join condition: None + +(247) Project [codegen id : 58] +Output [6]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Input [8]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, i_item_sk#91] + +(248) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#92] + +(249) BroadcastHashJoin [codegen id : 58] +Left keys [1]: [ws_promo_sk#77] +Right keys [1]: [p_promo_sk#92] +Join type: Inner +Join condition: None + +(250) Project [codegen id : 58] +Output [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Input [7]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, p_promo_sk#92] + +(251) HashAggregate [codegen id : 58] +Input [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] +Keys [1]: [web_site_id#90] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#79)), partial_sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] +Aggregate Attributes [5]: [sum#207, sum#208, isEmpty#209, sum#210, isEmpty#211] +Results [6]: [web_site_id#90, sum#212, sum#213, isEmpty#214, sum#215, isEmpty#216] + +(252) Exchange +Input [6]: [web_site_id#90, sum#212, sum#213, isEmpty#214, sum#215, isEmpty#216] +Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=41] + +(253) HashAggregate [codegen id : 59] +Input [6]: [web_site_id#90, sum#212, sum#213, isEmpty#214, sum#215, isEmpty#216] +Keys [1]: [web_site_id#90] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#79)), sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#79))#103, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105] +Results [5]: [web channel AS channel#106, concat(web_site, web_site_id#90) AS id#107, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#79))#103,17,2) AS sales#108, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104 AS returns#109, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105 AS profit#110] + +(254) Union + +(255) HashAggregate [codegen id : 60] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116] +Results [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] + +(256) Exchange +Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=42] + +(257) HashAggregate [codegen id : 61] Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#123, sum(returns#37)#124, sum(profit#38)#125] -Results [3]: [sum(sales#36)#123 AS sales#129, sum(returns#37)#124 AS returns#130, sum(profit#38)#125 AS profit#131] +Results [3]: [sum(sales#36)#123 AS sales#162, sum(returns#37)#124 AS returns#163, sum(profit#38)#125 AS profit#164] -(110) HashAggregate [codegen id : 61] -Input [3]: [sales#129, returns#130, profit#131] +(258) HashAggregate [codegen id : 61] +Input [3]: [sales#162, returns#163, profit#164] Keys: [] -Functions [3]: [partial_sum(sales#129), partial_sum(returns#130), partial_sum(profit#131)] -Aggregate Attributes [6]: [sum#151, isEmpty#152, sum#153, isEmpty#154, sum#155, isEmpty#156] -Results [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] +Functions [3]: [partial_sum(sales#162), partial_sum(returns#163), partial_sum(profit#164)] +Aggregate Attributes [6]: [sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222] +Results [6]: [sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228] -(111) Exchange -Input [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=17] +(259) Exchange +Input [6]: [sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=43] -(112) HashAggregate [codegen id : 62] -Input [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] +(260) HashAggregate [codegen id : 62] +Input [6]: [sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228] Keys: [] -Functions [3]: [sum(sales#129), sum(returns#130), sum(profit#131)] -Aggregate Attributes [3]: [sum(sales#129)#163, sum(returns#130)#164, sum(profit#131)#165] -Results [5]: [null AS channel#166, null AS id#167, sum(sales#129)#163 AS sales#168, sum(returns#130)#164 AS returns#169, sum(profit#131)#165 AS profit#170] +Functions [3]: [sum(sales#162), sum(returns#163), sum(profit#164)] +Aggregate Attributes [3]: [sum(sales#162)#229, sum(returns#163)#230, sum(profit#164)#231] +Results [5]: [null AS channel#232, null AS id#233, sum(sales#162)#229 AS sales#234, sum(returns#163)#230 AS returns#235, sum(profit#164)#231 AS profit#236] -(113) Union +(261) Union -(114) HashAggregate [codegen id : 63] +(262) HashAggregate [codegen id : 63] Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Keys [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Functions: [] Aggregate Attributes: [] Results [5]: [channel#34, id#35, sales#126, returns#127, profit#128] -(115) Exchange +(263) Exchange Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] -Arguments: hashpartitioning(channel#34, id#35, sales#126, returns#127, profit#128, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(channel#34, id#35, sales#126, returns#127, profit#128, 5), ENSURE_REQUIREMENTS, [plan_id=44] -(116) HashAggregate [codegen id : 64] +(264) HashAggregate [codegen id : 64] Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Keys [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Functions: [] Aggregate Attributes: [] Results [5]: [channel#34, id#35, sales#126, returns#127, profit#128] -(117) TakeOrderedAndProject +(265) TakeOrderedAndProject Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#126, returns#127, profit#128] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (122) -+- * ColumnarToRow (121) - +- CometProject (120) - +- CometFilter (119) - +- CometScan parquet spark_catalog.default.date_dim (118) +BroadcastExchange (270) ++- * ColumnarToRow (269) + +- CometProject (268) + +- CometFilter (267) + +- CometScan parquet spark_catalog.default.date_dim (266) (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#171] +Output [2]: [d_date_sk#14, d_date#237] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(119) CometFilter -Input [2]: [d_date_sk#14, d_date#171] -Condition : (((isnotnull(d_date#171) AND (d_date#171 >= 1998-08-04)) AND (d_date#171 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) +(267) CometFilter +Input [2]: [d_date_sk#14, d_date#237] +Condition : (((isnotnull(d_date#237) AND (d_date#237 >= 1998-08-04)) AND (d_date#237 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) -(120) CometProject -Input [2]: [d_date_sk#14, d_date#171] +(268) CometProject +Input [2]: [d_date_sk#14, d_date#237] Arguments: [d_date_sk#14], [d_date_sk#14] -(121) ColumnarToRow [codegen id : 1] +(269) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(122) BroadcastExchange +(270) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=45] Subquery:2 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 Subquery:3 Hosting operator id = 69 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#8 +Subquery:4 Hosting operator id = 103 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 + +Subquery:5 Hosting operator id = 127 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 + +Subquery:6 Hosting operator id = 151 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#8 + +Subquery:7 Hosting operator id = 182 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 + +Subquery:8 Hosting operator id = 206 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 + +Subquery:9 Hosting operator id = 230 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#8 + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt index f6a37c6372..332c26b97e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt @@ -165,13 +165,219 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + Exchange [channel,id] #19 + WholeStageCodegen (39) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (26) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [s_store_id] #20 + WholeStageCodegen (25) + HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_ticket_number] + CometExchange [ss_item_sk,ss_ticket_number] #21 + CometFilter [ss_store_sk,ss_item_sk,ss_promo_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] #6 + InputAdapter + ReusedExchange [d_date_sk] #5 + InputAdapter + ReusedExchange [s_store_sk,s_store_id] #7 + InputAdapter + ReusedExchange [i_item_sk] #8 + InputAdapter + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (32) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [cp_catalog_page_id] #22 + WholeStageCodegen (31) + HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + BroadcastHashJoin [cs_item_sk,i_item_sk] + Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] + Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + ColumnarToRow + InputAdapter + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number] + CometExchange [cs_item_sk,cs_order_number] #23 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #12 + InputAdapter + ReusedExchange [d_date_sk] #5 + InputAdapter + ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + InputAdapter + ReusedExchange [i_item_sk] #8 + InputAdapter + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (38) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [web_site_id] #24 + WholeStageCodegen (37) + HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + BroadcastHashJoin [ws_promo_sk,p_promo_sk] + Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + BroadcastHashJoin [ws_item_sk,i_item_sk] + Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_order_number] + CometExchange [ws_item_sk,ws_order_number] #25 + CometFilter [ws_web_site_sk,ws_item_sk,ws_promo_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] #16 + InputAdapter + ReusedExchange [d_date_sk] #5 + InputAdapter + ReusedExchange [web_site_sk,web_site_id] #17 + InputAdapter + ReusedExchange [i_item_sk] #8 + InputAdapter + ReusedExchange [p_promo_sk] #9 WholeStageCodegen (62) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter - Exchange #19 + Exchange #26 WholeStageCodegen (61) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + Exchange [channel,id] #27 + WholeStageCodegen (60) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (47) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [s_store_id] #28 + WholeStageCodegen (46) + HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_ticket_number] + CometExchange [ss_item_sk,ss_ticket_number] #29 + CometFilter [ss_store_sk,ss_item_sk,ss_promo_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] #6 + InputAdapter + ReusedExchange [d_date_sk] #5 + InputAdapter + ReusedExchange [s_store_sk,s_store_id] #7 + InputAdapter + ReusedExchange [i_item_sk] #8 + InputAdapter + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (53) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [cp_catalog_page_id] #30 + WholeStageCodegen (52) + HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + BroadcastHashJoin [cs_item_sk,i_item_sk] + Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] + Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + ColumnarToRow + InputAdapter + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number] + CometExchange [cs_item_sk,cs_order_number] #31 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #12 + InputAdapter + ReusedExchange [d_date_sk] #5 + InputAdapter + ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + InputAdapter + ReusedExchange [i_item_sk] #8 + InputAdapter + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (59) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [web_site_id] #32 + WholeStageCodegen (58) + HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + BroadcastHashJoin [ws_promo_sk,p_promo_sk] + Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + BroadcastHashJoin [ws_item_sk,i_item_sk] + Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_order_number] + CometExchange [ws_item_sk,ws_order_number] #33 + CometFilter [ws_web_site_sk,ws_item_sk,ws_promo_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] #16 + InputAdapter + ReusedExchange [d_date_sk] #5 + InputAdapter + ReusedExchange [web_site_sk,web_site_id] #17 + InputAdapter + ReusedExchange [i_item_sk] #8 + InputAdapter + ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index e5b3523dcf..383eca8ec6 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateMode import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometCollectLimitExec, CometFilterExec, CometHashAggregateExec, CometHashJoinExec, CometProjectExec, CometRowToColumnarExec, CometScanExec, CometSortExec, CometSortMergeJoinExec, CometTakeOrderedAndProjectExec} import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometShuffleExchangeExec} import org.apache.spark.sql.execution.{CollectLimitExec, ProjectExec, SQLExecution, UnionExec} -import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec +import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{BroadcastNestedLoopJoinExec, CartesianProductExec, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.expressions.Window @@ -62,6 +62,37 @@ class CometExecSuite extends CometTestBase { } } + test("CometShuffleExchangeExec logical link should be correct") { + withTempView("v") { + spark.sparkContext + .parallelize((1 to 4).map(i => TestData(i, i.toString)), 2) + .toDF("c1", "c2") + .createOrReplaceTempView("v") + + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key -> "true") { + val df = sql("SELECT * FROM v where c1 = 1 order by c1, c2") + val shuffle = find(df.queryExecution.executedPlan) { + case _: CometShuffleExchangeExec => true + case _ => false + }.get.asInstanceOf[CometShuffleExchangeExec] + assert(shuffle.logicalLink.isEmpty) + } + + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key -> "false") { + val df = sql("SELECT * FROM v where c1 = 1 order by c1, c2") + val shuffle = find(df.queryExecution.executedPlan) { + case _: ShuffleExchangeExec => true + case _ => false + }.get.asInstanceOf[ShuffleExchangeExec] + assert(shuffle.logicalLink.isEmpty) + } + } + } + test("Ensure that the correct outputPartitioning of CometSort") { withTable("test_data") { val tableDF = spark.sparkContext @@ -302,7 +333,8 @@ class CometExecSuite extends CometTestBase { withSQLConf( CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true", - "spark.sql.autoBroadcastJoinThreshold" -> "0", + "spark.sql.adaptive.autoBroadcastJoinThreshold" -> "-1", + "spark.sql.autoBroadcastJoinThreshold" -> "-1", "spark.sql.join.preferSortMergeJoin" -> "true") { withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl1") { withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl2") { @@ -373,6 +405,7 @@ class CometExecSuite extends CometTestBase { withSQLConf( SQLConf.EXCHANGE_REUSE_ENABLED.key -> "true", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key -> "true") { withTable(tableName, dim) { @@ -1306,3 +1339,5 @@ case class BucketedTableTestSpec( expectedShuffle: Boolean = true, expectedSort: Boolean = true, expectedNumOutputPartitions: Option[Int] = None) + +case class TestData(key: Int, value: String) From d67071e33a5cfca81a700a58eed55d16a348e93f Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 25 Apr 2024 12:59:57 -0700 Subject: [PATCH 2/8] Implement equals and hashCode for CometShuffleExchangeExec --- .../shuffle/CometShuffleExchangeExec.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala index faa52cd06b..e05ac7a23a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala @@ -50,6 +50,8 @@ import org.apache.spark.util.MutablePair import org.apache.spark.util.collection.unsafe.sort.{PrefixComparators, RecordComparator} import org.apache.spark.util.random.XORShiftRandom +import com.google.common.base.Objects + import org.apache.comet.serde.{OperatorOuterClass, PartitioningOuterClass, QueryPlanSerde} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.QueryPlanSerde.serializeDataType @@ -193,6 +195,21 @@ case class CometShuffleExchangeExec( override protected def withNewChildInternal(newChild: SparkPlan): CometShuffleExchangeExec = copy(child = newChild) + + override def equals(obj: Any): Boolean = { + obj match { + case other: CometShuffleExchangeExec => + this.outputPartitioning == other.outputPartitioning && + this.shuffleOrigin == other.shuffleOrigin && this.child == other.child && + this.shuffleType == other.shuffleType && + this.advisoryPartitionSize == other.advisoryPartitionSize + case _ => + false + } + } + + override def hashCode(): Int = + Objects.hashCode(outputPartitioning, shuffleOrigin, shuffleType, advisoryPartitionSize, child) } object CometShuffleExchangeExec extends ShimCometShuffleExchangeExec { From fbccbf5c8c408493fc126fe00345166019676d20 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 25 Apr 2024 13:07:09 -0700 Subject: [PATCH 3/8] Update plan stability --- .../approved-plans-v1_4/q23b/explain.txt | 238 ++--- .../approved-plans-v1_4/q23b/simplified.txt | 21 +- .../approved-plans-v1_4/q44/explain.txt | 233 ++--- .../approved-plans-v1_4/q44/simplified.txt | 14 +- .../approved-plans-v1_4/q64/explain.txt | 542 +++++----- .../approved-plans-v1_4/q64/simplified.txt | 15 +- .../approved-plans-v1_4/q75/explain.txt | 441 ++++---- .../approved-plans-v1_4/q75/simplified.txt | 19 +- .../approved-plans-v1_4/q95/explain.txt | 323 +++--- .../approved-plans-v1_4/q95/simplified.txt | 15 +- .../approved-plans-v2_7/q64/explain.txt | 542 +++++----- .../approved-plans-v2_7/q64/simplified.txt | 15 +- .../approved-plans-v2_7/q75/explain.txt | 441 ++++---- .../approved-plans-v2_7/q75/simplified.txt | 19 +- .../approved-plans-v2_7/q80a/explain.txt | 978 ++---------------- .../approved-plans-v2_7/q80a/simplified.txt | 212 +--- 16 files changed, 1283 insertions(+), 2785 deletions(-) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt index f1b720650f..d60f0c4837 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt @@ -1,6 +1,6 @@ == Physical Plan == -TakeOrderedAndProject (98) -+- Union (97) +TakeOrderedAndProject (87) ++- Union (86) :- * HashAggregate (62) : +- Exchange (61) : +- * HashAggregate (60) @@ -63,13 +63,13 @@ TakeOrderedAndProject (98) : : +- * HashAggregate (49) : : +- ReusedExchange (48) : +- ReusedExchange (57) - +- * HashAggregate (96) - +- Exchange (95) - +- * HashAggregate (94) - +- * Project (93) - +- * BroadcastHashJoin Inner BuildRight (92) - :- * Project (90) - : +- * BroadcastHashJoin Inner BuildRight (89) + +- * HashAggregate (85) + +- Exchange (84) + +- * HashAggregate (83) + +- * Project (82) + +- * BroadcastHashJoin Inner BuildRight (81) + :- * Project (79) + : +- * BroadcastHashJoin Inner BuildRight (78) : :- * SortMergeJoin LeftSemi (76) : : :- * Sort (70) : : : +- Exchange (69) @@ -84,19 +84,8 @@ TakeOrderedAndProject (98) : : +- * Filter (73) : : +- * HashAggregate (72) : : +- ReusedExchange (71) - : +- BroadcastExchange (88) - : +- * SortMergeJoin LeftSemi (87) - : :- * ColumnarToRow (81) - : : +- CometSort (80) - : : +- CometExchange (79) - : : +- CometFilter (78) - : : +- CometScan parquet spark_catalog.default.customer (77) - : +- * Sort (86) - : +- * Project (85) - : +- * Filter (84) - : +- * HashAggregate (83) - : +- ReusedExchange (82) - +- ReusedExchange (91) + : +- ReusedExchange (77) + +- ReusedExchange (80) (unknown) Scan parquet spark_catalog.default.catalog_sales @@ -129,7 +118,7 @@ Condition : isnotnull(ss_item_sk#7) (6) ColumnarToRow [codegen id : 3] Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -(7) ReusedExchange [Reuses operator id: 108] +(7) ReusedExchange [Reuses operator id: 97] Output [2]: [d_date_sk#10, d_date#11] (8) BroadcastHashJoin [codegen id : 3] @@ -364,7 +353,7 @@ Join condition: None Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#34, c_last_name#35] Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#33, c_first_name#34, c_last_name#35] -(57) ReusedExchange [Reuses operator id: 103] +(57) ReusedExchange [Reuses operator id: 92] Output [1]: [d_date_sk#36] (58) BroadcastHashJoin [codegen id : 16] @@ -459,115 +448,64 @@ Right keys [1]: [c_customer_sk#24] Join type: LeftSemi Join condition: None -(unknown) Scan parquet spark_catalog.default.customer +(77) ReusedExchange [Reuses operator id: 54] Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(78) CometFilter -Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] -Condition : isnotnull(c_customer_sk#49) - -(79) CometExchange -Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] -Arguments: hashpartitioning(c_customer_sk#49, 5), Exchange hashpartitioning(c_customer_sk#49, 5), ENSURE_REQUIREMENTS, [plan_id=12], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] - -(80) CometSort -Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] -Arguments: [c_customer_sk#49, c_first_name#50, c_last_name#51], [c_customer_sk#49 ASC NULLS FIRST] - -(81) ColumnarToRow [codegen id : 27] -Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] - -(82) ReusedExchange [Reuses operator id: 37] -Output [3]: [c_customer_sk#24, sum#27, isEmpty#28] - -(83) HashAggregate [codegen id : 30] -Input [3]: [c_customer_sk#24, sum#27, isEmpty#28] -Keys [1]: [c_customer_sk#24] -Functions [1]: [sum((cast(ss_quantity#21 as decimal(10,0)) * ss_sales_price#22))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#21 as decimal(10,0)) * ss_sales_price#22))#29] -Results [2]: [c_customer_sk#24, sum((cast(ss_quantity#21 as decimal(10,0)) * ss_sales_price#22))#29 AS ssales#30] -(84) Filter [codegen id : 30] -Input [2]: [c_customer_sk#24, ssales#30] -Condition : (isnotnull(ssales#30) AND (cast(ssales#30 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#31, [id=#32]))) - -(85) Project [codegen id : 30] -Output [1]: [c_customer_sk#24] -Input [2]: [c_customer_sk#24, ssales#30] - -(86) Sort [codegen id : 30] -Input [1]: [c_customer_sk#24] -Arguments: [c_customer_sk#24 ASC NULLS FIRST], false, 0 - -(87) SortMergeJoin [codegen id : 31] -Left keys [1]: [c_customer_sk#49] -Right keys [1]: [c_customer_sk#24] -Join type: LeftSemi -Join condition: None - -(88) BroadcastExchange -Input [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(89) BroadcastHashJoin [codegen id : 33] +(78) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ws_bill_customer_sk#44] Right keys [1]: [c_customer_sk#49] Join type: Inner Join condition: None -(90) Project [codegen id : 33] +(79) Project [codegen id : 33] Output [5]: [ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47, c_first_name#50, c_last_name#51] Input [7]: [ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47, c_customer_sk#49, c_first_name#50, c_last_name#51] -(91) ReusedExchange [Reuses operator id: 103] +(80) ReusedExchange [Reuses operator id: 92] Output [1]: [d_date_sk#52] -(92) BroadcastHashJoin [codegen id : 33] +(81) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ws_sold_date_sk#47] Right keys [1]: [d_date_sk#52] Join type: Inner Join condition: None -(93) Project [codegen id : 33] +(82) Project [codegen id : 33] Output [4]: [ws_quantity#45, ws_list_price#46, c_first_name#50, c_last_name#51] Input [6]: [ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47, c_first_name#50, c_last_name#51, d_date_sk#52] -(94) HashAggregate [codegen id : 33] +(83) HashAggregate [codegen id : 33] Input [4]: [ws_quantity#45, ws_list_price#46, c_first_name#50, c_last_name#51] Keys [2]: [c_last_name#51, c_first_name#50] Functions [1]: [partial_sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))] Aggregate Attributes [2]: [sum#53, isEmpty#54] Results [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -(95) Exchange +(84) Exchange Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=12] -(96) HashAggregate [codegen id : 34] +(85) HashAggregate [codegen id : 34] Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] Keys [2]: [c_last_name#51, c_first_name#50] Functions [1]: [sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))] Aggregate Attributes [1]: [sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))#57] Results [3]: [c_last_name#51, c_first_name#50, sum((cast(ws_quantity#45 as decimal(10,0)) * ws_list_price#46))#57 AS sales#58] -(97) Union +(86) Union -(98) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [3]: [c_last_name#35, c_first_name#34, sales#42] Arguments: 100, [c_last_name#35 ASC NULLS FIRST, c_first_name#34 ASC NULLS FIRST, sales#42 ASC NULLS FIRST], [c_last_name#35, c_first_name#34, sales#42] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (103) -+- * ColumnarToRow (102) - +- CometProject (101) - +- CometFilter (100) - +- CometScan parquet spark_catalog.default.date_dim (99) +BroadcastExchange (92) ++- * ColumnarToRow (91) + +- CometProject (90) + +- CometFilter (89) + +- CometScan parquet spark_catalog.default.date_dim (88) (unknown) Scan parquet spark_catalog.default.date_dim @@ -577,27 +515,27 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(100) CometFilter +(89) CometFilter Input [3]: [d_date_sk#36, d_year#59, d_moy#60] Condition : ((((isnotnull(d_year#59) AND isnotnull(d_moy#60)) AND (d_year#59 = 2000)) AND (d_moy#60 = 2)) AND isnotnull(d_date_sk#36)) -(101) CometProject +(90) CometProject Input [3]: [d_date_sk#36, d_year#59, d_moy#60] Arguments: [d_date_sk#36], [d_date_sk#36] -(102) ColumnarToRow [codegen id : 1] +(91) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#36] -(103) BroadcastExchange +(92) BroadcastExchange Input [1]: [d_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] Subquery:2 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (108) -+- * ColumnarToRow (107) - +- CometProject (106) - +- CometFilter (105) - +- CometScan parquet spark_catalog.default.date_dim (104) +BroadcastExchange (97) ++- * ColumnarToRow (96) + +- CometProject (95) + +- CometFilter (94) + +- CometScan parquet spark_catalog.default.date_dim (93) (unknown) Scan parquet spark_catalog.default.date_dim @@ -607,37 +545,37 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(105) CometFilter +(94) CometFilter Input [3]: [d_date_sk#10, d_date#11, d_year#61] Condition : (d_year#61 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) -(106) CometProject +(95) CometProject Input [3]: [d_date_sk#10, d_date#11, d_year#61] Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] -(107) ColumnarToRow [codegen id : 1] +(96) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#10, d_date#11] -(108) BroadcastExchange +(97) BroadcastExchange Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] Subquery:3 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* HashAggregate (123) -+- Exchange (122) - +- * HashAggregate (121) - +- * HashAggregate (120) - +- Exchange (119) - +- * HashAggregate (118) - +- * Project (117) - +- * BroadcastHashJoin Inner BuildRight (116) - :- * Project (114) - : +- * BroadcastHashJoin Inner BuildRight (113) - : :- * ColumnarToRow (111) - : : +- CometFilter (110) - : : +- CometScan parquet spark_catalog.default.store_sales (109) - : +- ReusedExchange (112) - +- ReusedExchange (115) +* HashAggregate (112) ++- Exchange (111) + +- * HashAggregate (110) + +- * HashAggregate (109) + +- Exchange (108) + +- * HashAggregate (107) + +- * Project (106) + +- * BroadcastHashJoin Inner BuildRight (105) + :- * Project (103) + : +- * BroadcastHashJoin Inner BuildRight (102) + : :- * ColumnarToRow (100) + : : +- CometFilter (99) + : : +- CometScan parquet spark_catalog.default.store_sales (98) + : +- ReusedExchange (101) + +- ReusedExchange (104) (unknown) Scan parquet spark_catalog.default.store_sales @@ -648,81 +586,81 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#65), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(110) CometFilter +(99) CometFilter Input [4]: [ss_customer_sk#62, ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65] Condition : isnotnull(ss_customer_sk#62) -(111) ColumnarToRow [codegen id : 3] +(100) ColumnarToRow [codegen id : 3] Input [4]: [ss_customer_sk#62, ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65] -(112) ReusedExchange [Reuses operator id: 33] +(101) ReusedExchange [Reuses operator id: 33] Output [1]: [c_customer_sk#67] -(113) BroadcastHashJoin [codegen id : 3] +(102) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_customer_sk#62] Right keys [1]: [c_customer_sk#67] Join type: Inner Join condition: None -(114) Project [codegen id : 3] +(103) Project [codegen id : 3] Output [4]: [ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65, c_customer_sk#67] Input [5]: [ss_customer_sk#62, ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65, c_customer_sk#67] -(115) ReusedExchange [Reuses operator id: 128] +(104) ReusedExchange [Reuses operator id: 117] Output [1]: [d_date_sk#68] -(116) BroadcastHashJoin [codegen id : 3] +(105) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_sold_date_sk#65] Right keys [1]: [d_date_sk#68] Join type: Inner Join condition: None -(117) Project [codegen id : 3] +(106) Project [codegen id : 3] Output [3]: [ss_quantity#63, ss_sales_price#64, c_customer_sk#67] Input [5]: [ss_quantity#63, ss_sales_price#64, ss_sold_date_sk#65, c_customer_sk#67, d_date_sk#68] -(118) HashAggregate [codegen id : 3] +(107) HashAggregate [codegen id : 3] Input [3]: [ss_quantity#63, ss_sales_price#64, c_customer_sk#67] Keys [1]: [c_customer_sk#67] Functions [1]: [partial_sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))] Aggregate Attributes [2]: [sum#69, isEmpty#70] Results [3]: [c_customer_sk#67, sum#71, isEmpty#72] -(119) Exchange +(108) Exchange Input [3]: [c_customer_sk#67, sum#71, isEmpty#72] -Arguments: hashpartitioning(c_customer_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(c_customer_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(120) HashAggregate [codegen id : 4] +(109) HashAggregate [codegen id : 4] Input [3]: [c_customer_sk#67, sum#71, isEmpty#72] Keys [1]: [c_customer_sk#67] Functions [1]: [sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))] Aggregate Attributes [1]: [sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))#73] Results [1]: [sum((cast(ss_quantity#63 as decimal(10,0)) * ss_sales_price#64))#73 AS csales#74] -(121) HashAggregate [codegen id : 4] +(110) HashAggregate [codegen id : 4] Input [1]: [csales#74] Keys: [] Functions [1]: [partial_max(csales#74)] Aggregate Attributes [1]: [max#75] Results [1]: [max#76] -(122) Exchange +(111) Exchange Input [1]: [max#76] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=19] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] -(123) HashAggregate [codegen id : 5] +(112) HashAggregate [codegen id : 5] Input [1]: [max#76] Keys: [] Functions [1]: [max(csales#74)] Aggregate Attributes [1]: [max(csales#74)#77] Results [1]: [max(csales#74)#77 AS tpcds_cmax#78] -Subquery:4 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#65 IN dynamicpruning#66 -BroadcastExchange (128) -+- * ColumnarToRow (127) - +- CometProject (126) - +- CometFilter (125) - +- CometScan parquet spark_catalog.default.date_dim (124) +Subquery:4 Hosting operator id = 98 Hosting Expression = ss_sold_date_sk#65 IN dynamicpruning#66 +BroadcastExchange (117) ++- * ColumnarToRow (116) + +- CometProject (115) + +- CometFilter (114) + +- CometScan parquet spark_catalog.default.date_dim (113) (unknown) Scan parquet spark_catalog.default.date_dim @@ -732,20 +670,20 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(125) CometFilter +(114) CometFilter Input [2]: [d_date_sk#68, d_year#79] Condition : (d_year#79 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#68)) -(126) CometProject +(115) CometProject Input [2]: [d_date_sk#68, d_year#79] Arguments: [d_date_sk#68], [d_date_sk#68] -(127) ColumnarToRow [codegen id : 1] +(116) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#68] -(128) BroadcastExchange +(117) BroadcastExchange Input [1]: [d_date_sk#68] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] Subquery:5 Hosting operator id = 50 Hosting Expression = ReusedSubquery Subquery scalar-subquery#31, [id=#32] @@ -753,6 +691,4 @@ Subquery:6 Hosting operator id = 63 Hosting Expression = ws_sold_date_sk#47 IN d Subquery:7 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#31, [id=#32] -Subquery:8 Hosting operator id = 84 Hosting Expression = ReusedSubquery Subquery scalar-subquery#31, [id=#32] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt index c53ac73588..9cbb568a9a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt @@ -181,25 +181,6 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] InputAdapter ReusedExchange [c_customer_sk,sum,isEmpty] #8 InputAdapter - BroadcastExchange #17 - WholeStageCodegen (31) - SortMergeJoin [c_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (27) - ColumnarToRow - InputAdapter - CometSort [c_customer_sk] - CometExchange [c_customer_sk] #18 - CometFilter [c_customer_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - WholeStageCodegen (30) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #3 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #8 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #13 InputAdapter ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt index 37d9690f93..f0e9cc19d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (30) - : : +- * SortMergeJoin Inner (29) +TakeOrderedAndProject (33) ++- * Project (32) + +- * BroadcastHashJoin Inner BuildRight (31) + :- * Project (29) + : +- * BroadcastHashJoin Inner BuildRight (28) + : :- * Project (23) + : : +- * SortMergeJoin Inner (22) : : :- * Sort (14) : : : +- * Project (13) : : : +- * Filter (12) @@ -20,25 +20,18 @@ TakeOrderedAndProject (40) : : : +- CometProject (3) : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- * Sort (28) - : : +- * Project (27) - : : +- * Filter (26) - : : +- Window (25) - : : +- * ColumnarToRow (24) - : : +- CometSort (23) - : : +- CometExchange (22) - : : +- CometFilter (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.store_sales (15) - : +- BroadcastExchange (34) - : +- * ColumnarToRow (33) - : +- CometFilter (32) - : +- CometScan parquet spark_catalog.default.item (31) - +- ReusedExchange (37) + : : +- * Sort (21) + : : +- * Project (20) + : : +- * Filter (19) + : : +- Window (18) + : : +- * ColumnarToRow (17) + : : +- CometSort (16) + : : +- ReusedExchange (15) + : +- BroadcastExchange (27) + : +- * ColumnarToRow (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.item (24) + +- ReusedExchange (30) (unknown) Scan parquet spark_catalog.default.store_sales @@ -101,165 +94,129 @@ Input [3]: [item_sk#7, rank_col#8, rnk#11] Input [2]: [item_sk#7, rnk#11] Arguments: [rnk#11 ASC NULLS FIRST], false, 0 -(unknown) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#12, ss_store_sk#13, ss_net_profit#14, ss_sold_date_sk#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] -ReadSchema: struct - -(16) CometFilter -Input [4]: [ss_item_sk#12, ss_store_sk#13, ss_net_profit#14, ss_sold_date_sk#15] -Condition : (isnotnull(ss_store_sk#13) AND (ss_store_sk#13 = 4)) - -(17) CometProject -Input [4]: [ss_item_sk#12, ss_store_sk#13, ss_net_profit#14, ss_sold_date_sk#15] -Arguments: [ss_item_sk#12, ss_net_profit#14], [ss_item_sk#12, ss_net_profit#14] +(15) ReusedExchange [Reuses operator id: 8] +Output [2]: [item_sk#12, rank_col#13] -(18) CometHashAggregate -Input [2]: [ss_item_sk#12, ss_net_profit#14] -Keys [1]: [ss_item_sk#12] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#14))] +(16) CometSort +Input [2]: [item_sk#12, rank_col#13] +Arguments: [item_sk#12, rank_col#13], [rank_col#13 DESC NULLS LAST] -(19) CometExchange -Input [3]: [ss_item_sk#12, sum#16, count#17] -Arguments: hashpartitioning(ss_item_sk#12, 5), Exchange hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(17) ColumnarToRow [codegen id : 3] +Input [2]: [item_sk#12, rank_col#13] -(20) CometHashAggregate -Input [3]: [ss_item_sk#12, sum#16, count#17] -Keys [1]: [ss_item_sk#12] -Functions [1]: [avg(UnscaledValue(ss_net_profit#14))] +(18) Window +Input [2]: [item_sk#12, rank_col#13] +Arguments: [rank(rank_col#13) windowspecdefinition(rank_col#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#13 DESC NULLS LAST] -(21) CometFilter -Input [2]: [item_sk#18, rank_col#19] -Condition : (isnotnull(rank_col#19) AND (cast(rank_col#19 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) +(19) Filter [codegen id : 4] +Input [3]: [item_sk#12, rank_col#13, rnk#14] +Condition : ((rnk#14 < 11) AND isnotnull(item_sk#12)) -(22) CometExchange -Input [2]: [item_sk#18, rank_col#19] -Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(20) Project [codegen id : 4] +Output [2]: [item_sk#12, rnk#14] +Input [3]: [item_sk#12, rank_col#13, rnk#14] -(23) CometSort -Input [2]: [item_sk#18, rank_col#19] -Arguments: [item_sk#18, rank_col#19], [rank_col#19 DESC NULLS LAST] +(21) Sort [codegen id : 4] +Input [2]: [item_sk#12, rnk#14] +Arguments: [rnk#14 ASC NULLS FIRST], false, 0 -(24) ColumnarToRow [codegen id : 3] -Input [2]: [item_sk#18, rank_col#19] - -(25) Window -Input [2]: [item_sk#18, rank_col#19] -Arguments: [rank(rank_col#19) windowspecdefinition(rank_col#19 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#20], [rank_col#19 DESC NULLS LAST] - -(26) Filter [codegen id : 4] -Input [3]: [item_sk#18, rank_col#19, rnk#20] -Condition : ((rnk#20 < 11) AND isnotnull(item_sk#18)) - -(27) Project [codegen id : 4] -Output [2]: [item_sk#18, rnk#20] -Input [3]: [item_sk#18, rank_col#19, rnk#20] - -(28) Sort [codegen id : 4] -Input [2]: [item_sk#18, rnk#20] -Arguments: [rnk#20 ASC NULLS FIRST], false, 0 - -(29) SortMergeJoin [codegen id : 7] +(22) SortMergeJoin [codegen id : 7] Left keys [1]: [rnk#11] -Right keys [1]: [rnk#20] +Right keys [1]: [rnk#14] Join type: Inner Join condition: None -(30) Project [codegen id : 7] -Output [3]: [item_sk#7, rnk#11, item_sk#18] -Input [4]: [item_sk#7, rnk#11, item_sk#18, rnk#20] +(23) Project [codegen id : 7] +Output [3]: [item_sk#7, rnk#11, item_sk#12] +Input [4]: [item_sk#7, rnk#11, item_sk#12, rnk#14] (unknown) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#21, i_product_name#22] +Output [2]: [i_item_sk#15, i_product_name#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(32) CometFilter -Input [2]: [i_item_sk#21, i_product_name#22] -Condition : isnotnull(i_item_sk#21) +(25) CometFilter +Input [2]: [i_item_sk#15, i_product_name#16] +Condition : isnotnull(i_item_sk#15) -(33) ColumnarToRow [codegen id : 5] -Input [2]: [i_item_sk#21, i_product_name#22] +(26) ColumnarToRow [codegen id : 5] +Input [2]: [i_item_sk#15, i_product_name#16] -(34) BroadcastExchange -Input [2]: [i_item_sk#21, i_product_name#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(27) BroadcastExchange +Input [2]: [i_item_sk#15, i_product_name#16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(35) BroadcastHashJoin [codegen id : 7] +(28) BroadcastHashJoin [codegen id : 7] Left keys [1]: [item_sk#7] -Right keys [1]: [i_item_sk#21] +Right keys [1]: [i_item_sk#15] Join type: Inner Join condition: None -(36) Project [codegen id : 7] -Output [3]: [rnk#11, item_sk#18, i_product_name#22] -Input [5]: [item_sk#7, rnk#11, item_sk#18, i_item_sk#21, i_product_name#22] +(29) Project [codegen id : 7] +Output [3]: [rnk#11, item_sk#12, i_product_name#16] +Input [5]: [item_sk#7, rnk#11, item_sk#12, i_item_sk#15, i_product_name#16] -(37) ReusedExchange [Reuses operator id: 34] -Output [2]: [i_item_sk#23, i_product_name#24] +(30) ReusedExchange [Reuses operator id: 27] +Output [2]: [i_item_sk#17, i_product_name#18] -(38) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [item_sk#18] -Right keys [1]: [i_item_sk#23] +(31) BroadcastHashJoin [codegen id : 7] +Left keys [1]: [item_sk#12] +Right keys [1]: [i_item_sk#17] Join type: Inner Join condition: None -(39) Project [codegen id : 7] -Output [3]: [rnk#11, i_product_name#22 AS best_performing#25, i_product_name#24 AS worst_performing#26] -Input [5]: [rnk#11, item_sk#18, i_product_name#22, i_item_sk#23, i_product_name#24] +(32) Project [codegen id : 7] +Output [3]: [rnk#11, i_product_name#16 AS best_performing#19, i_product_name#18 AS worst_performing#20] +Input [5]: [rnk#11, item_sk#12, i_product_name#16, i_item_sk#17, i_product_name#18] -(40) TakeOrderedAndProject -Input [3]: [rnk#11, best_performing#25, worst_performing#26] -Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#25, worst_performing#26] +(33) TakeOrderedAndProject +Input [3]: [rnk#11, best_performing#19, worst_performing#20] +Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#19, worst_performing#20] ===== Subqueries ===== Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (47) -+- CometHashAggregate (46) - +- CometExchange (45) - +- CometHashAggregate (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan parquet spark_catalog.default.store_sales (41) +* ColumnarToRow (40) ++- CometHashAggregate (39) + +- CometExchange (38) + +- CometHashAggregate (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan parquet spark_catalog.default.store_sales (34) (unknown) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_addr_sk#27, ss_store_sk#28, ss_net_profit#29, ss_sold_date_sk#30] +Output [4]: [ss_addr_sk#21, ss_store_sk#22, ss_net_profit#23, ss_sold_date_sk#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(42) CometFilter -Input [4]: [ss_addr_sk#27, ss_store_sk#28, ss_net_profit#29, ss_sold_date_sk#30] -Condition : ((isnotnull(ss_store_sk#28) AND (ss_store_sk#28 = 4)) AND isnull(ss_addr_sk#27)) - -(43) CometProject -Input [4]: [ss_addr_sk#27, ss_store_sk#28, ss_net_profit#29, ss_sold_date_sk#30] -Arguments: [ss_store_sk#28, ss_net_profit#29], [ss_store_sk#28, ss_net_profit#29] +(35) CometFilter +Input [4]: [ss_addr_sk#21, ss_store_sk#22, ss_net_profit#23, ss_sold_date_sk#24] +Condition : ((isnotnull(ss_store_sk#22) AND (ss_store_sk#22 = 4)) AND isnull(ss_addr_sk#21)) -(44) CometHashAggregate -Input [2]: [ss_store_sk#28, ss_net_profit#29] -Keys [1]: [ss_store_sk#28] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#29))] +(36) CometProject +Input [4]: [ss_addr_sk#21, ss_store_sk#22, ss_net_profit#23, ss_sold_date_sk#24] +Arguments: [ss_store_sk#22, ss_net_profit#23], [ss_store_sk#22, ss_net_profit#23] -(45) CometExchange -Input [3]: [ss_store_sk#28, sum#31, count#32] -Arguments: hashpartitioning(ss_store_sk#28, 5), Exchange hashpartitioning(ss_store_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=10], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +(37) CometHashAggregate +Input [2]: [ss_store_sk#22, ss_net_profit#23] +Keys [1]: [ss_store_sk#22] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#23))] -(46) CometHashAggregate -Input [3]: [ss_store_sk#28, sum#31, count#32] -Keys [1]: [ss_store_sk#28] -Functions [1]: [avg(UnscaledValue(ss_net_profit#29))] +(38) CometExchange +Input [3]: [ss_store_sk#22, sum#25, count#26] +Arguments: hashpartitioning(ss_store_sk#22, 5), Exchange hashpartitioning(ss_store_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(47) ColumnarToRow [codegen id : 1] -Input [1]: [rank_col#33] +(39) CometHashAggregate +Input [3]: [ss_store_sk#22, sum#25, count#26] +Keys [1]: [ss_store_sk#22] +Functions [1]: [avg(UnscaledValue(ss_net_profit#23))] -Subquery:2 Hosting operator id = 21 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +(40) ColumnarToRow [codegen id : 1] +Input [1]: [rank_col#27] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt index a1a70deade..bc065345c8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt @@ -46,21 +46,13 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] ColumnarToRow InputAdapter CometSort [rank_col] - CometExchange #4 - CometFilter [rank_col] - ReusedSubquery [rank_col] #1 - CometHashAggregate [ss_item_sk,sum,count] - CometExchange [ss_item_sk] #5 - CometHashAggregate [ss_item_sk,ss_net_profit] - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_store_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + ReusedExchange [item_sk,rank_col] #1 InputAdapter - BroadcastExchange #6 + BroadcastExchange #4 WholeStageCodegen (5) ColumnarToRow InputAdapter CometFilter [i_item_sk] CometScan parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter - ReusedExchange [i_item_sk,i_product_name] #6 + ReusedExchange [i_item_sk,i_product_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt index 8404fe47fa..c7a7f93d5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* Sort (192) -+- Exchange (191) - +- * Project (190) - +- * SortMergeJoin Inner (189) +* Sort (179) ++- Exchange (178) + +- * Project (177) + +- * SortMergeJoin Inner (176) :- * Sort (108) : +- Exchange (107) : +- * HashAggregate (106) @@ -111,43 +111,43 @@ : +- CometProject (100) : +- CometFilter (99) : +- CometScan parquet spark_catalog.default.item (98) - +- * Sort (188) - +- Exchange (187) - +- * HashAggregate (186) - +- * HashAggregate (185) - +- * Project (184) - +- * BroadcastHashJoin Inner BuildRight (183) - :- * Project (181) - : +- * BroadcastHashJoin Inner BuildRight (180) - : :- * Project (178) - : : +- * BroadcastHashJoin Inner BuildRight (177) - : : :- * Project (175) - : : : +- * BroadcastHashJoin Inner BuildRight (174) - : : : :- * Project (172) - : : : : +- * BroadcastHashJoin Inner BuildRight (171) - : : : : :- * Project (169) - : : : : : +- * BroadcastHashJoin Inner BuildRight (168) - : : : : : :- * Project (166) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (165) - : : : : : : :- * Project (163) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (162) - : : : : : : : :- * Project (160) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (159) - : : : : : : : : :- * Project (157) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (156) - : : : : : : : : : :- * Project (154) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) - : : : : : : : : : : :- * Project (151) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) - : : : : : : : : : : : :- * Project (148) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) - : : : : : : : : : : : : :- * Project (145) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) - : : : : : : : : : : : : : :- * Project (142) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) - : : : : : : : : : : : : : : :- * ColumnarToRow (139) - : : : : : : : : : : : : : : : +- CometProject (138) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (137) + +- * Sort (175) + +- Exchange (174) + +- * HashAggregate (173) + +- * HashAggregate (172) + +- * Project (171) + +- * BroadcastHashJoin Inner BuildRight (170) + :- * Project (168) + : +- * BroadcastHashJoin Inner BuildRight (167) + : :- * Project (165) + : : +- * BroadcastHashJoin Inner BuildRight (164) + : : :- * Project (162) + : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : :- * Project (159) + : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : :- * Project (156) + : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : :- * Project (153) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : :- * Project (150) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : :- * Project (147) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : :- * Project (144) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : :- * Project (141) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : :- * Project (129) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) + : : : : : : : : : : : : : : :- * ColumnarToRow (126) + : : : : : : : : : : : : : : : +- CometProject (125) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (124) : : : : : : : : : : : : : : : :- CometSort (118) : : : : : : : : : : : : : : : : +- CometExchange (117) : : : : : : : : : : : : : : : : +- CometProject (116) @@ -158,39 +158,26 @@ : : : : : : : : : : : : : : : : +- CometProject (114) : : : : : : : : : : : : : : : : +- CometFilter (113) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (112) - : : : : : : : : : : : : : : : +- CometSort (136) - : : : : : : : : : : : : : : : +- CometProject (135) - : : : : : : : : : : : : : : : +- CometFilter (134) - : : : : : : : : : : : : : : : +- CometHashAggregate (133) - : : : : : : : : : : : : : : : +- CometExchange (132) - : : : : : : : : : : : : : : : +- CometHashAggregate (131) - : : : : : : : : : : : : : : : +- CometProject (130) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (129) - : : : : : : : : : : : : : : : :- CometSort (123) - : : : : : : : : : : : : : : : : +- CometExchange (122) - : : : : : : : : : : : : : : : : +- CometProject (121) - : : : : : : : : : : : : : : : : +- CometFilter (120) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (119) - : : : : : : : : : : : : : : : +- CometSort (128) - : : : : : : : : : : : : : : : +- CometExchange (127) - : : : : : : : : : : : : : : : +- CometProject (126) - : : : : : : : : : : : : : : : +- CometFilter (125) - : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (124) - : : : : : : : : : : : : : : +- ReusedExchange (140) - : : : : : : : : : : : : : +- ReusedExchange (143) - : : : : : : : : : : : : +- ReusedExchange (146) - : : : : : : : : : : : +- ReusedExchange (149) - : : : : : : : : : : +- ReusedExchange (152) - : : : : : : : : : +- ReusedExchange (155) - : : : : : : : : +- ReusedExchange (158) - : : : : : : : +- ReusedExchange (161) - : : : : : : +- ReusedExchange (164) - : : : : : +- ReusedExchange (167) - : : : : +- ReusedExchange (170) - : : : +- ReusedExchange (173) - : : +- ReusedExchange (176) - : +- ReusedExchange (179) - +- ReusedExchange (182) + : : : : : : : : : : : : : : : +- CometSort (123) + : : : : : : : : : : : : : : : +- CometProject (122) + : : : : : : : : : : : : : : : +- CometFilter (121) + : : : : : : : : : : : : : : : +- CometHashAggregate (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : +- ReusedExchange (145) + : : : : : : : +- ReusedExchange (148) + : : : : : : +- ReusedExchange (151) + : : : : : +- ReusedExchange (154) + : : : : +- ReusedExchange (157) + : : : +- ReusedExchange (160) + : : +- ReusedExchange (163) + : +- ReusedExchange (166) + +- ReusedExchange (169) (unknown) Scan parquet spark_catalog.default.store_sales @@ -334,7 +321,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (31) ColumnarToRow [codegen id : 16] Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(32) ReusedExchange [Reuses operator id: 196] +(32) ReusedExchange [Reuses operator id: 183] Output [2]: [d_date_sk#32, d_year#33] (33) BroadcastHashJoin [codegen id : 16] @@ -738,341 +725,280 @@ Arguments: hashpartitioning(ss_item_sk#101, 5), Exchange hashpartitioning(ss_ite Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101 ASC NULLS FIRST] -(unknown) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] -ReadSchema: struct - -(120) CometFilter -Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] -Condition : (isnotnull(cs_item_sk#117) AND isnotnull(cs_order_number#118)) - -(121) CometProject -Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] -Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] - -(122) CometExchange -Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] -Arguments: hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), Exchange hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] - -(123) CometSort -Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] -Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117 ASC NULLS FIRST, cs_order_number#118 ASC NULLS FIRST] - -(unknown) Scan parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(125) CometFilter -Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] -Condition : (isnotnull(cr_item_sk#121) AND isnotnull(cr_order_number#122)) - -(126) CometProject -Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] -Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] - -(127) CometExchange -Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), Exchange hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), ENSURE_REQUIREMENTS, [plan_id=23], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=24] +(119) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] -(128) CometSort -Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121 ASC NULLS FIRST, cr_order_number#122 ASC NULLS FIRST] - -(129) CometSortMergeJoin -Left output [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] -Right output [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: [cs_item_sk#117, cs_order_number#118], [cr_item_sk#121, cr_order_number#122], Inner - -(130) CometProject -Input [8]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] - -(131) CometHashAggregate -Input [5]: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +(120) CometHashAggregate +Input [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] Keys [1]: [cs_item_sk#117] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#119)), partial_sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] - -(132) CometExchange -Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] -Arguments: hashpartitioning(cs_item_sk#117, 5), Exchange hashpartitioning(cs_item_sk#117, 5), ENSURE_REQUIREMENTS, [plan_id=25], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=26] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#121)), sum(((cr_refunded_cash#122 + cr_reversed_charge#123) + cr_store_credit#124))] -(133) CometHashAggregate -Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] -Keys [1]: [cs_item_sk#117] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#119)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] - -(134) CometFilter +(121) CometFilter Input [3]: [cs_item_sk#117, sale#30, refund#31] Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) -(135) CometProject +(122) CometProject Input [3]: [cs_item_sk#117, sale#30, refund#31] Arguments: [cs_item_sk#117], [cs_item_sk#117] -(136) CometSort +(123) CometSort Input [1]: [cs_item_sk#117] Arguments: [cs_item_sk#117], [cs_item_sk#117 ASC NULLS FIRST] -(137) CometSortMergeJoin +(124) CometSortMergeJoin Left output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Right output [1]: [cs_item_sk#117] Arguments: [ss_item_sk#101], [cs_item_sk#117], Inner -(138) CometProject +(125) CometProject Input [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, cs_item_sk#117] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(139) ColumnarToRow [codegen id : 33] +(126) ColumnarToRow [codegen id : 33] Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(140) ReusedExchange [Reuses operator id: 200] -Output [2]: [d_date_sk#130, d_year#131] +(127) ReusedExchange [Reuses operator id: 187] +Output [2]: [d_date_sk#125, d_year#126] -(141) BroadcastHashJoin [codegen id : 33] +(128) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_sold_date_sk#112] -Right keys [1]: [d_date_sk#130] +Right keys [1]: [d_date_sk#125] Join type: Inner Join condition: None -(142) Project [codegen id : 33] -Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131] -Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#130, d_year#131] +(129) Project [codegen id : 33] +Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126] +Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#125, d_year#126] -(143) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#132, s_store_name#133, s_zip#134] +(130) ReusedExchange [Reuses operator id: 38] +Output [3]: [s_store_sk#127, s_store_name#128, s_zip#129] -(144) BroadcastHashJoin [codegen id : 33] +(131) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_store_sk#106] -Right keys [1]: [s_store_sk#132] +Right keys [1]: [s_store_sk#127] Join type: Inner Join condition: None -(145) Project [codegen id : 33] -Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134] -Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_sk#132, s_store_name#133, s_zip#134] +(132) Project [codegen id : 33] +Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129] +Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_sk#127, s_store_name#128, s_zip#129] -(146) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] +(133) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] -(147) BroadcastHashJoin [codegen id : 33] +(134) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_customer_sk#102] -Right keys [1]: [c_customer_sk#135] +Right keys [1]: [c_customer_sk#130] Join type: Inner Join condition: None -(148) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] -Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] +(135) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] +Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] -(149) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#141, d_year#142] +(136) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#136, d_year#137] -(150) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_sales_date_sk#140] -Right keys [1]: [d_date_sk#141] +(137) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_sales_date_sk#135] +Right keys [1]: [d_date_sk#136] Join type: Inner Join condition: None -(151) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140, d_date_sk#141, d_year#142] +(138) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135, d_date_sk#136, d_year#137] -(152) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#143, d_year#144] +(139) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#138, d_year#139] -(153) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_shipto_date_sk#139] -Right keys [1]: [d_date_sk#143] +(140) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_shipto_date_sk#134] +Right keys [1]: [d_date_sk#138] Join type: Inner Join condition: None -(154) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142, d_date_sk#143, d_year#144] +(141) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137, d_date_sk#138, d_year#139] -(155) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#145, cd_marital_status#146] +(142) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#140, cd_marital_status#141] -(156) BroadcastHashJoin [codegen id : 33] +(143) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_cdemo_sk#103] -Right keys [1]: [cd_demo_sk#145] +Right keys [1]: [cd_demo_sk#140] Join type: Inner Join condition: None -(157) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_demo_sk#145, cd_marital_status#146] +(144) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_demo_sk#140, cd_marital_status#141] -(158) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#147, cd_marital_status#148] +(145) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#142, cd_marital_status#143] -(159) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_cdemo_sk#136] -Right keys [1]: [cd_demo_sk#147] +(146) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_cdemo_sk#131] +Right keys [1]: [cd_demo_sk#142] Join type: Inner -Join condition: NOT (cd_marital_status#146 = cd_marital_status#148) +Join condition: NOT (cd_marital_status#141 = cd_marital_status#143) -(160) Project [codegen id : 33] -Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] -Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146, cd_demo_sk#147, cd_marital_status#148] +(147) Project [codegen id : 33] +Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] +Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141, cd_demo_sk#142, cd_marital_status#143] -(161) ReusedExchange [Reuses operator id: 68] -Output [1]: [p_promo_sk#149] +(148) ReusedExchange [Reuses operator id: 68] +Output [1]: [p_promo_sk#144] -(162) BroadcastHashJoin [codegen id : 33] +(149) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_promo_sk#107] -Right keys [1]: [p_promo_sk#149] +Right keys [1]: [p_promo_sk#144] Join type: Inner Join condition: None -(163) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, p_promo_sk#149] +(150) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, p_promo_sk#144] -(164) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] +(151) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#145, hd_income_band_sk#146] -(165) BroadcastHashJoin [codegen id : 33] +(152) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_hdemo_sk#104] -Right keys [1]: [hd_demo_sk#150] +Right keys [1]: [hd_demo_sk#145] Join type: Inner Join condition: None -(166) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_demo_sk#150, hd_income_band_sk#151] +(153) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_demo_sk#145, hd_income_band_sk#146] -(167) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] +(154) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#147, hd_income_band_sk#148] -(168) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_hdemo_sk#137] -Right keys [1]: [hd_demo_sk#152] +(155) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_hdemo_sk#132] +Right keys [1]: [hd_demo_sk#147] Join type: Inner Join condition: None -(169) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153] -Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_demo_sk#152, hd_income_band_sk#153] +(156) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148] +Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_demo_sk#147, hd_income_band_sk#148] -(170) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(157) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] -(171) BroadcastHashJoin [codegen id : 33] +(158) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_addr_sk#105] -Right keys [1]: [ca_address_sk#154] +Right keys [1]: [ca_address_sk#149] Join type: Inner Join condition: None -(172) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(159) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] +Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] -(173) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +(160) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(174) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_addr_sk#138] -Right keys [1]: [ca_address_sk#159] +(161) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_addr_sk#133] +Right keys [1]: [ca_address_sk#154] Join type: Inner Join condition: None -(175) Project [codegen id : 33] -Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +(162) Project [codegen id : 33] +Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(176) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#164] +(163) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#159] -(177) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#151] -Right keys [1]: [ib_income_band_sk#164] +(164) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#146] +Right keys [1]: [ib_income_band_sk#159] Join type: Inner Join condition: None -(178) Project [codegen id : 33] -Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#164] +(165) Project [codegen id : 33] +Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#159] -(179) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#165] +(166) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#160] -(180) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#153] -Right keys [1]: [ib_income_band_sk#165] +(167) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#148] +Right keys [1]: [ib_income_band_sk#160] Join type: Inner Join condition: None -(181) Project [codegen id : 33] -Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#165] +(168) Project [codegen id : 33] +Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#160] -(182) ReusedExchange [Reuses operator id: 102] -Output [2]: [i_item_sk#166, i_product_name#167] +(169) ReusedExchange [Reuses operator id: 102] +Output [2]: [i_item_sk#161, i_product_name#162] -(183) BroadcastHashJoin [codegen id : 33] +(170) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_item_sk#101] -Right keys [1]: [i_item_sk#166] +Right keys [1]: [i_item_sk#161] Join type: Inner Join condition: None -(184) Project [codegen id : 33] -Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] +(171) Project [codegen id : 33] +Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] -(185) HashAggregate [codegen id : 33] -Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] -Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] +(172) HashAggregate [codegen id : 33] +Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] +Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#109)), partial_sum(UnscaledValue(ss_list_price#110)), partial_sum(UnscaledValue(ss_coupon_amt#111))] -Aggregate Attributes [4]: [count#72, sum#168, sum#169, sum#170] -Results [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] +Aggregate Attributes [4]: [count#72, sum#163, sum#164, sum#165] +Results [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] -(186) HashAggregate [codegen id : 33] -Input [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] -Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] +(173) HashAggregate [codegen id : 33] +Input [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] +Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#109)), sum(UnscaledValue(ss_list_price#110)), sum(UnscaledValue(ss_coupon_amt#111))] Aggregate Attributes [4]: [count(1)#80, sum(UnscaledValue(ss_wholesale_cost#109))#81, sum(UnscaledValue(ss_list_price#110))#82, sum(UnscaledValue(ss_coupon_amt#111))#83] -Results [8]: [i_item_sk#166 AS item_sk#174, s_store_name#133 AS store_name#175, s_zip#134 AS store_zip#176, d_year#131 AS syear#177, count(1)#80 AS cnt#178, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#179, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#180, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#181] +Results [8]: [i_item_sk#161 AS item_sk#169, s_store_name#128 AS store_name#170, s_zip#129 AS store_zip#171, d_year#126 AS syear#172, count(1)#80 AS cnt#173, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#174, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#175, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#176] -(187) Exchange -Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] -Arguments: hashpartitioning(item_sk#174, store_name#175, store_zip#176, 5), ENSURE_REQUIREMENTS, [plan_id=27] +(174) Exchange +Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] +Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=21] -(188) Sort [codegen id : 34] -Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] -Arguments: [item_sk#174 ASC NULLS FIRST, store_name#175 ASC NULLS FIRST, store_zip#176 ASC NULLS FIRST], false, 0 +(175) Sort [codegen id : 34] +Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] +Arguments: [item_sk#169 ASC NULLS FIRST, store_name#170 ASC NULLS FIRST, store_zip#171 ASC NULLS FIRST], false, 0 -(189) SortMergeJoin [codegen id : 35] +(176) SortMergeJoin [codegen id : 35] Left keys [3]: [item_sk#85, store_name#86, store_zip#87] -Right keys [3]: [item_sk#174, store_name#175, store_zip#176] +Right keys [3]: [item_sk#169, store_name#170, store_zip#171] Join type: Inner -Join condition: (cnt#178 <= cnt#97) +Join condition: (cnt#173 <= cnt#97) -(190) Project [codegen id : 35] -Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] -Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] +(177) Project [codegen id : 35] +Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] +Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -(191) Exchange -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] -Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=28] +(178) Exchange +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] +Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=22] -(192) Sort [codegen id : 36] -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] -Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST], true, 0 +(179) Sort [codegen id : 36] +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] +Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST], true, 0 ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (196) -+- * ColumnarToRow (195) - +- CometFilter (194) - +- CometScan parquet spark_catalog.default.date_dim (193) +BroadcastExchange (183) ++- * ColumnarToRow (182) + +- CometFilter (181) + +- CometScan parquet spark_catalog.default.date_dim (180) (unknown) Scan parquet spark_catalog.default.date_dim @@ -1082,40 +1008,40 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(194) CometFilter +(181) CometFilter Input [2]: [d_date_sk#32, d_year#33] Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) -(195) ColumnarToRow [codegen id : 1] +(182) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#32, d_year#33] -(196) BroadcastExchange +(183) BroadcastExchange Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=29] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=23] Subquery:2 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#112 IN dynamicpruning#113 -BroadcastExchange (200) -+- * ColumnarToRow (199) - +- CometFilter (198) - +- CometScan parquet spark_catalog.default.date_dim (197) +BroadcastExchange (187) ++- * ColumnarToRow (186) + +- CometFilter (185) + +- CometScan parquet spark_catalog.default.date_dim (184) (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#130, d_year#131] +Output [2]: [d_date_sk#125, d_year#126] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(198) CometFilter -Input [2]: [d_date_sk#130, d_year#131] -Condition : ((isnotnull(d_year#131) AND (d_year#131 = 2000)) AND isnotnull(d_date_sk#130)) +(185) CometFilter +Input [2]: [d_date_sk#125, d_year#126] +Condition : ((isnotnull(d_year#126) AND (d_year#126 = 2000)) AND isnotnull(d_date_sk#125)) -(199) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#130, d_year#131] +(186) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#125, d_year#126] -(200) BroadcastExchange -Input [2]: [d_date_sk#130, d_year#131] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=30] +(187) BroadcastExchange +Input [2]: [d_date_sk#125, d_year#126] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt index 5b6f72526d..eda7db4e57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt @@ -221,20 +221,7 @@ WholeStageCodegen (36) CometProject [cs_item_sk] CometFilter [sale,refund] CometHashAggregate [cs_item_sk,sum,sum,isEmpty] - CometExchange [cs_item_sk] #22 - CometHashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number] - CometExchange [cs_item_sk,cs_order_number] #23 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #24 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #21 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index 62b89d9d95..5c7ccc79eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -TakeOrderedAndProject (141) -+- * Project (140) - +- * SortMergeJoin Inner (139) +TakeOrderedAndProject (132) ++- * Project (131) + +- * SortMergeJoin Inner (130) :- * Sort (71) : +- Exchange (70) : +- * Filter (69) @@ -73,18 +73,18 @@ TakeOrderedAndProject (141) : +- CometProject (56) : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.web_returns (54) - +- * Sort (138) - +- Exchange (137) - +- * Filter (136) - +- * HashAggregate (135) - +- Exchange (134) - +- * HashAggregate (133) - +- * HashAggregate (132) - +- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * Project (90) - : +- * SortMergeJoin LeftOuter (89) + +- * Sort (129) + +- Exchange (128) + +- * Filter (127) + +- * HashAggregate (126) + +- Exchange (125) + +- * HashAggregate (124) + +- * HashAggregate (123) + +- Exchange (122) + +- * HashAggregate (121) + +- Union (120) + :- * Project (87) + : +- * SortMergeJoin LeftOuter (86) : :- * Sort (82) : : +- Exchange (81) : : +- * Project (80) @@ -96,50 +96,41 @@ TakeOrderedAndProject (141) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) : : : +- ReusedExchange (75) : : +- ReusedExchange (78) - : +- * ColumnarToRow (88) - : +- CometSort (87) - : +- CometExchange (86) - : +- CometProject (85) - : +- CometFilter (84) - : +- CometScan parquet spark_catalog.default.catalog_returns (83) - :- * Project (109) - : +- * SortMergeJoin LeftOuter (108) - : :- * Sort (101) - : : +- Exchange (100) - : : +- * Project (99) - : : +- * BroadcastHashJoin Inner BuildRight (98) - : : :- * Project (96) - : : : +- * BroadcastHashJoin Inner BuildRight (95) - : : : :- * ColumnarToRow (93) - : : : : +- CometFilter (92) - : : : : +- CometScan parquet spark_catalog.default.store_sales (91) - : : : +- ReusedExchange (94) - : : +- ReusedExchange (97) - : +- * ColumnarToRow (107) - : +- CometSort (106) - : +- CometExchange (105) - : +- CometProject (104) - : +- CometFilter (103) - : +- CometScan parquet spark_catalog.default.store_returns (102) - +- * Project (128) - +- * SortMergeJoin LeftOuter (127) - :- * Sort (120) - : +- Exchange (119) - : +- * Project (118) - : +- * BroadcastHashJoin Inner BuildRight (117) - : :- * Project (115) - : : +- * BroadcastHashJoin Inner BuildRight (114) - : : :- * ColumnarToRow (112) - : : : +- CometFilter (111) - : : : +- CometScan parquet spark_catalog.default.web_sales (110) - : : +- ReusedExchange (113) - : +- ReusedExchange (116) - +- * ColumnarToRow (126) - +- CometSort (125) - +- CometExchange (124) - +- CometProject (123) - +- CometFilter (122) - +- CometScan parquet spark_catalog.default.web_returns (121) + : +- * ColumnarToRow (85) + : +- CometSort (84) + : +- ReusedExchange (83) + :- * Project (103) + : +- * SortMergeJoin LeftOuter (102) + : :- * Sort (98) + : : +- Exchange (97) + : : +- * Project (96) + : : +- * BroadcastHashJoin Inner BuildRight (95) + : : :- * Project (93) + : : : +- * BroadcastHashJoin Inner BuildRight (92) + : : : :- * ColumnarToRow (90) + : : : : +- CometFilter (89) + : : : : +- CometScan parquet spark_catalog.default.store_sales (88) + : : : +- ReusedExchange (91) + : : +- ReusedExchange (94) + : +- * ColumnarToRow (101) + : +- CometSort (100) + : +- ReusedExchange (99) + +- * Project (119) + +- * SortMergeJoin LeftOuter (118) + :- * Sort (114) + : +- Exchange (113) + : +- * Project (112) + : +- * BroadcastHashJoin Inner BuildRight (111) + : :- * Project (109) + : : +- * BroadcastHashJoin Inner BuildRight (108) + : : :- * ColumnarToRow (106) + : : : +- CometFilter (105) + : : : +- CometScan parquet spark_catalog.default.web_sales (104) + : : +- ReusedExchange (107) + : +- ReusedExchange (110) + +- * ColumnarToRow (117) + +- CometSort (116) + +- ReusedExchange (115) (unknown) Scan parquet spark_catalog.default.catalog_sales @@ -189,7 +180,7 @@ Join condition: None Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -(11) ReusedExchange [Reuses operator id: 145] +(11) ReusedExchange [Reuses operator id: 136] Output [2]: [d_date_sk#13, d_year#14] (12) BroadcastHashJoin [codegen id : 3] @@ -274,7 +265,7 @@ Join condition: None Output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -(30) ReusedExchange [Reuses operator id: 145] +(30) ReusedExchange [Reuses operator id: 136] Output [2]: [d_date_sk#33, d_year#34] (31) BroadcastHashJoin [codegen id : 9] @@ -359,7 +350,7 @@ Join condition: None Output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -(49) ReusedExchange [Reuses operator id: 145] +(49) ReusedExchange [Reuses operator id: 136] Output [2]: [d_date_sk#53, d_year#54] (50) BroadcastHashJoin [codegen id : 15] @@ -494,7 +485,7 @@ Join condition: None Output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Input [10]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(78) ReusedExchange [Reuses operator id: 149] +(78) ReusedExchange [Reuses operator id: 140] Output [2]: [d_date_sk#81, d_year#82] (79) BroadcastHashJoin [codegen id : 25] @@ -515,283 +506,235 @@ Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIR Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Arguments: [cs_order_number#71 ASC NULLS FIRST, cs_item_sk#70 ASC NULLS FIRST], false, 0 -(unknown) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(84) CometFilter -Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] -Condition : (isnotnull(cr_order_number#84) AND isnotnull(cr_item_sk#83)) +(83) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(85) CometProject -Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] -Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] - -(86) CometExchange -Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -Arguments: hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), Exchange hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] - -(87) CometSort +(84) CometSort Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_order_number#84 ASC NULLS FIRST, cr_item_sk#83 ASC NULLS FIRST] -(88) ColumnarToRow [codegen id : 27] +(85) ColumnarToRow [codegen id : 27] Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(89) SortMergeJoin [codegen id : 28] +(86) SortMergeJoin [codegen id : 28] Left keys [2]: [cs_order_number#71, cs_item_sk#70] Right keys [2]: [cr_order_number#84, cr_item_sk#83] Join type: LeftOuter Join condition: None -(90) Project [codegen id : 28] +(87) Project [codegen id : 28] Output [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, (cs_quantity#72 - coalesce(cr_return_quantity#85, 0)) AS sales_cnt#20, (cs_ext_sales_price#73 - coalesce(cr_return_amount#86, 0.00)) AS sales_amt#21] Input [13]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82, cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] (unknown) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] +Output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#92), dynamicpruningexpression(ss_sold_date_sk#92 IN dynamicpruning#93)] +PartitionFilters: [isnotnull(ss_sold_date_sk#91), dynamicpruningexpression(ss_sold_date_sk#91 IN dynamicpruning#92)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(92) CometFilter -Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] -Condition : isnotnull(ss_item_sk#88) +(89) CometFilter +Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +Condition : isnotnull(ss_item_sk#87) -(93) ColumnarToRow [codegen id : 31] -Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] +(90) ColumnarToRow [codegen id : 31] +Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -(94) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] +(91) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -(95) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_item_sk#88] -Right keys [1]: [i_item_sk#94] +(92) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_item_sk#87] +Right keys [1]: [i_item_sk#93] Join type: Inner Join condition: None -(96) Project [codegen id : 31] -Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] -Input [10]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] +(93) Project [codegen id : 31] +Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +Input [10]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -(97) ReusedExchange [Reuses operator id: 149] -Output [2]: [d_date_sk#99, d_year#100] +(94) ReusedExchange [Reuses operator id: 140] +Output [2]: [d_date_sk#98, d_year#99] -(98) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_sold_date_sk#92] -Right keys [1]: [d_date_sk#99] +(95) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_sold_date_sk#91] +Right keys [1]: [d_date_sk#98] Join type: Inner Join condition: None -(99) Project [codegen id : 31] -Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] -Input [11]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_date_sk#99, d_year#100] - -(100) Exchange -Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] -Arguments: hashpartitioning(ss_ticket_number#89, ss_item_sk#88, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(101) Sort [codegen id : 32] -Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] -Arguments: [ss_ticket_number#89 ASC NULLS FIRST, ss_item_sk#88 ASC NULLS FIRST], false, 0 - -(unknown) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct +(96) Project [codegen id : 31] +Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_date_sk#98, d_year#99] -(103) CometFilter -Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] -Condition : (isnotnull(sr_ticket_number#102) AND isnotnull(sr_item_sk#101)) +(97) Exchange +Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(104) CometProject -Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] -Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +(98) Sort [codegen id : 32] +Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Arguments: [ss_ticket_number#88 ASC NULLS FIRST, ss_item_sk#87 ASC NULLS FIRST], false, 0 -(105) CometExchange -Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] -Arguments: hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), Exchange hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=18], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=19] +(99) ReusedExchange [Reuses operator id: 38] +Output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -(106) CometSort -Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] -Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_ticket_number#102 ASC NULLS FIRST, sr_item_sk#101 ASC NULLS FIRST] +(100) CometSort +Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +Arguments: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103], [sr_ticket_number#101 ASC NULLS FIRST, sr_item_sk#100 ASC NULLS FIRST] -(107) ColumnarToRow [codegen id : 33] -Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +(101) ColumnarToRow [codegen id : 33] +Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -(108) SortMergeJoin [codegen id : 34] -Left keys [2]: [ss_ticket_number#89, ss_item_sk#88] -Right keys [2]: [sr_ticket_number#102, sr_item_sk#101] +(102) SortMergeJoin [codegen id : 34] +Left keys [2]: [ss_ticket_number#88, ss_item_sk#87] +Right keys [2]: [sr_ticket_number#101, sr_item_sk#100] Join type: LeftOuter Join condition: None -(109) Project [codegen id : 34] -Output [7]: [d_year#100, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, (ss_quantity#90 - coalesce(sr_return_quantity#103, 0)) AS sales_cnt#40, (ss_ext_sales_price#91 - coalesce(sr_return_amt#104, 0.00)) AS sales_amt#41] -Input [13]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100, sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +(103) Project [codegen id : 34] +Output [7]: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, (ss_quantity#89 - coalesce(sr_return_quantity#102, 0)) AS sales_cnt#40, (ss_ext_sales_price#90 - coalesce(sr_return_amt#103, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99, sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] (unknown) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +Output [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#110), dynamicpruningexpression(ws_sold_date_sk#110 IN dynamicpruning#111)] +PartitionFilters: [isnotnull(ws_sold_date_sk#108), dynamicpruningexpression(ws_sold_date_sk#108 IN dynamicpruning#109)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(111) CometFilter -Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -Condition : isnotnull(ws_item_sk#106) +(105) CometFilter +Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] +Condition : isnotnull(ws_item_sk#104) -(112) ColumnarToRow [codegen id : 37] -Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +(106) ColumnarToRow [codegen id : 37] +Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] -(113) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +(107) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] -(114) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_item_sk#106] -Right keys [1]: [i_item_sk#112] +(108) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_item_sk#104] +Right keys [1]: [i_item_sk#110] Join type: Inner Join condition: None -(115) Project [codegen id : 37] -Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Input [10]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +(109) Project [codegen id : 37] +Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] +Input [10]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] -(116) ReusedExchange [Reuses operator id: 149] -Output [2]: [d_date_sk#117, d_year#118] +(110) ReusedExchange [Reuses operator id: 140] +Output [2]: [d_date_sk#115, d_year#116] -(117) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_sold_date_sk#110] -Right keys [1]: [d_date_sk#117] +(111) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_sold_date_sk#108] +Right keys [1]: [d_date_sk#115] Join type: Inner Join condition: None -(118) Project [codegen id : 37] -Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Input [11]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_date_sk#117, d_year#118] - -(119) Exchange -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: hashpartitioning(ws_order_number#107, ws_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=20] - -(120) Sort [codegen id : 38] -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: [ws_order_number#107 ASC NULLS FIRST, ws_item_sk#106 ASC NULLS FIRST], false, 0 - -(unknown) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct +(112) Project [codegen id : 37] +Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] +Input [11]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_date_sk#115, d_year#116] -(122) CometFilter -Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] -Condition : (isnotnull(wr_order_number#120) AND isnotnull(wr_item_sk#119)) +(113) Exchange +Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] +Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=16] -(123) CometProject -Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] -Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +(114) Sort [codegen id : 38] +Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] +Arguments: [ws_order_number#105 ASC NULLS FIRST, ws_item_sk#104 ASC NULLS FIRST], false, 0 -(124) CometExchange -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), Exchange hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] +(115) ReusedExchange [Reuses operator id: 57] +Output [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -(125) CometSort -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_order_number#120 ASC NULLS FIRST, wr_item_sk#119 ASC NULLS FIRST] +(116) CometSort +Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +Arguments: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120], [wr_order_number#118 ASC NULLS FIRST, wr_item_sk#117 ASC NULLS FIRST] -(126) ColumnarToRow [codegen id : 39] -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +(117) ColumnarToRow [codegen id : 39] +Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -(127) SortMergeJoin [codegen id : 40] -Left keys [2]: [ws_order_number#107, ws_item_sk#106] -Right keys [2]: [wr_order_number#120, wr_item_sk#119] +(118) SortMergeJoin [codegen id : 40] +Left keys [2]: [ws_order_number#105, ws_item_sk#104] +Right keys [2]: [wr_order_number#118, wr_item_sk#117] Join type: LeftOuter Join condition: None -(128) Project [codegen id : 40] -Output [7]: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, (ws_quantity#108 - coalesce(wr_return_quantity#121, 0)) AS sales_cnt#60, (ws_ext_sales_price#109 - coalesce(wr_return_amt#122, 0.00)) AS sales_amt#61] -Input [13]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118, wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +(119) Project [codegen id : 40] +Output [7]: [d_year#116, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, (ws_quantity#106 - coalesce(wr_return_quantity#119, 0)) AS sales_cnt#60, (ws_ext_sales_price#107 - coalesce(wr_return_amt#120, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116, wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -(129) Union +(120) Union -(130) HashAggregate [codegen id : 41] +(121) HashAggregate [codegen id : 41] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(131) Exchange +(122) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=23] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=17] -(132) HashAggregate [codegen id : 42] +(123) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(133) HashAggregate [codegen id : 42] +(124) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -Aggregate Attributes [2]: [sum#62, sum#124] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] +Aggregate Attributes [2]: [sum#62, sum#121] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] -(134) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=24] +(125) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=18] -(135) HashAggregate [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] +(126) HashAggregate [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Aggregate Attributes [2]: [sum(sales_cnt#20)#66, sum(UnscaledValue(sales_amt#21))#67] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#126, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#127] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#123, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#124] -(136) Filter [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] -Condition : isnotnull(sales_cnt#126) +(127) Filter [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +Condition : isnotnull(sales_cnt#123) -(137) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=25] +(128) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=19] -(138) Sort [codegen id : 44] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +(129) Sort [codegen id : 44] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] Arguments: [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST], false, 0 -(139) SortMergeJoin [codegen id : 45] +(130) SortMergeJoin [codegen id : 45] Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Right keys [4]: [i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Join type: Inner -Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#126 as decimal(17,2))) < 0.90000000000000000000) +Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#123 as decimal(17,2))) < 0.90000000000000000000) -(140) Project [codegen id : 45] -Output [10]: [d_year#82 AS prev_year#128, d_year#14 AS year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#126 AS prev_yr_cnt#130, sales_cnt#68 AS curr_yr_cnt#131, (sales_cnt#68 - sales_cnt#126) AS sales_cnt_diff#132, (sales_amt#69 - sales_amt#127) AS sales_amt_diff#133] -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +(131) Project [codegen id : 45] +Output [10]: [d_year#82 AS prev_year#125, d_year#14 AS year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#123 AS prev_yr_cnt#127, sales_cnt#68 AS curr_yr_cnt#128, (sales_cnt#68 - sales_cnt#123) AS sales_cnt_diff#129, (sales_amt#69 - sales_amt#124) AS sales_amt_diff#130] +Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -(141) TakeOrderedAndProject -Input [10]: [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] -Arguments: 100, [sales_cnt_diff#132 ASC NULLS FIRST], [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] +(132) TakeOrderedAndProject +Input [10]: [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] +Arguments: 100, [sales_cnt_diff#129 ASC NULLS FIRST], [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (145) -+- * ColumnarToRow (144) - +- CometFilter (143) - +- CometScan parquet spark_catalog.default.date_dim (142) +BroadcastExchange (136) ++- * ColumnarToRow (135) + +- CometFilter (134) + +- CometScan parquet spark_catalog.default.date_dim (133) (unknown) Scan parquet spark_catalog.default.date_dim @@ -801,26 +744,26 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(143) CometFilter +(134) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(144) ColumnarToRow [codegen id : 1] +(135) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -(145) BroadcastExchange +(136) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=26] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=20] Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 -BroadcastExchange (149) -+- * ColumnarToRow (148) - +- CometFilter (147) - +- CometScan parquet spark_catalog.default.date_dim (146) +BroadcastExchange (140) ++- * ColumnarToRow (139) + +- CometFilter (138) + +- CometScan parquet spark_catalog.default.date_dim (137) (unknown) Scan parquet spark_catalog.default.date_dim @@ -830,19 +773,19 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(147) CometFilter +(138) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(148) ColumnarToRow [codegen id : 1] +(139) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -(149) BroadcastExchange +(140) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=27] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=21] -Subquery:5 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#92 IN dynamicpruning#75 +Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#75 -Subquery:6 Hosting operator id = 110 Hosting Expression = ws_sold_date_sk#110 IN dynamicpruning#75 +Subquery:6 Hosting operator id = 104 Hosting Expression = ws_sold_date_sk#108 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt index 6e9e0ea9e3..837398c8f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt @@ -178,10 +178,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [cr_order_number,cr_item_sk] - CometExchange [cr_order_number,cr_item_sk] #17 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_order_number,cr_item_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 WholeStageCodegen (34) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] @@ -189,7 +186,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat WholeStageCodegen (32) Sort [ss_ticket_number,ss_item_sk] InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #18 + Exchange [ss_ticket_number,ss_item_sk] #17 WholeStageCodegen (31) Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] @@ -209,10 +206,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [sr_ticket_number,sr_item_sk] - CometExchange [sr_ticket_number,sr_item_sk] #19 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_ticket_number,sr_item_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 WholeStageCodegen (40) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] @@ -220,7 +214,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat WholeStageCodegen (38) Sort [ws_order_number,ws_item_sk] InputAdapter - Exchange [ws_order_number,ws_item_sk] #20 + Exchange [ws_order_number,ws_item_sk] #18 WholeStageCodegen (37) Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] @@ -240,7 +234,4 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [wr_order_number,wr_item_sk] - CometExchange [wr_order_number,wr_item_sk] #21 - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_order_number,wr_item_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt index 49d1b9a85d..06488a31db 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt @@ -1,68 +1,65 @@ == Physical Plan == -* HashAggregate (64) -+- Exchange (63) - +- * HashAggregate (62) - +- * HashAggregate (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (52) - : +- * BroadcastHashJoin Inner BuildRight (51) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * SortMergeJoin LeftSemi (38) - : : : :- * SortMergeJoin LeftSemi (21) +* HashAggregate (61) ++- Exchange (60) + +- * HashAggregate (59) + +- * HashAggregate (58) + +- * HashAggregate (57) + +- * Project (56) + +- * BroadcastHashJoin Inner BuildRight (55) + :- * Project (49) + : +- * BroadcastHashJoin Inner BuildRight (48) + : :- * Project (42) + : : +- * BroadcastHashJoin Inner BuildRight (41) + : : :- * SortMergeJoin LeftSemi (35) + : : : :- * SortMergeJoin LeftSemi (18) : : : : :- * ColumnarToRow (6) : : : : : +- CometSort (5) : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (20) - : : : : +- * SortMergeJoin Inner (19) + : : : : +- * Project (17) + : : : : +- * SortMergeJoin Inner (16) : : : : :- * ColumnarToRow (12) : : : : : +- CometSort (11) : : : : : +- CometExchange (10) : : : : : +- CometProject (9) : : : : : +- CometFilter (8) : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) - : : : : +- * ColumnarToRow (18) - : : : : +- CometSort (17) - : : : : +- CometExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- * Project (37) - : : : +- * SortMergeJoin Inner (36) - : : : :- * ColumnarToRow (27) - : : : : +- CometSort (26) - : : : : +- CometExchange (25) - : : : : +- CometProject (24) - : : : : +- CometFilter (23) - : : : : +- CometScan parquet spark_catalog.default.web_returns (22) - : : : +- * Project (35) - : : : +- * SortMergeJoin Inner (34) - : : : :- * ColumnarToRow (30) - : : : : +- CometSort (29) - : : : : +- ReusedExchange (28) - : : : +- * ColumnarToRow (33) - : : : +- CometSort (32) - : : : +- ReusedExchange (31) - : : +- BroadcastExchange (43) - : : +- * ColumnarToRow (42) - : : +- CometProject (41) - : : +- CometFilter (40) - : : +- CometScan parquet spark_catalog.default.date_dim (39) - : +- BroadcastExchange (50) - : +- * ColumnarToRow (49) - : +- CometProject (48) - : +- CometFilter (47) - : +- CometScan parquet spark_catalog.default.customer_address (46) - +- BroadcastExchange (57) - +- * ColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan parquet spark_catalog.default.web_site (53) + : : : : +- * ColumnarToRow (15) + : : : : +- CometSort (14) + : : : : +- ReusedExchange (13) + : : : +- * Project (34) + : : : +- * SortMergeJoin Inner (33) + : : : :- * ColumnarToRow (24) + : : : : +- CometSort (23) + : : : : +- CometExchange (22) + : : : : +- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan parquet spark_catalog.default.web_returns (19) + : : : +- * Project (32) + : : : +- * SortMergeJoin Inner (31) + : : : :- * ColumnarToRow (27) + : : : : +- CometSort (26) + : : : : +- ReusedExchange (25) + : : : +- * ColumnarToRow (30) + : : : +- CometSort (29) + : : : +- ReusedExchange (28) + : : +- BroadcastExchange (40) + : : +- * ColumnarToRow (39) + : : +- CometProject (38) + : : +- CometFilter (37) + : : +- CometScan parquet spark_catalog.default.date_dim (36) + : +- BroadcastExchange (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometFilter (44) + : +- CometScan parquet spark_catalog.default.customer_address (43) + +- BroadcastExchange (54) + +- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) + +- CometScan parquet spark_catalog.default.web_site (50) (unknown) Scan parquet spark_catalog.default.web_sales @@ -117,245 +114,229 @@ Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS (12) ColumnarToRow [codegen id : 2] Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -(unknown) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_warehouse_sk#11, ws_order_number#12, ws_sold_date_sk#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] -ReadSchema: struct - -(14) CometFilter -Input [3]: [ws_warehouse_sk#11, ws_order_number#12, ws_sold_date_sk#13] -Condition : (isnotnull(ws_order_number#12) AND isnotnull(ws_warehouse_sk#11)) - -(15) CometProject -Input [3]: [ws_warehouse_sk#11, ws_order_number#12, ws_sold_date_sk#13] -Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_warehouse_sk#11, ws_order_number#12] - -(16) CometExchange -Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: hashpartitioning(ws_order_number#12, 5), Exchange hashpartitioning(ws_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(13) ReusedExchange [Reuses operator id: 10] +Output [2]: [ws_warehouse_sk#11, ws_order_number#12] -(17) CometSort +(14) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(18) ColumnarToRow [codegen id : 3] +(15) ColumnarToRow [codegen id : 3] Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -(19) SortMergeJoin [codegen id : 4] +(16) SortMergeJoin [codegen id : 4] Left keys [1]: [ws_order_number#9] Right keys [1]: [ws_order_number#12] Join type: Inner Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(20) Project [codegen id : 4] +(17) Project [codegen id : 4] Output [1]: [ws_order_number#9] Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] -(21) SortMergeJoin [codegen id : 5] +(18) SortMergeJoin [codegen id : 5] Left keys [1]: [ws_order_number#4] Right keys [1]: [ws_order_number#9] Join type: LeftSemi Join condition: None (unknown) Scan parquet spark_catalog.default.web_returns -Output [2]: [wr_order_number#14, wr_returned_date_sk#15] +Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number)] ReadSchema: struct -(23) CometFilter -Input [2]: [wr_order_number#14, wr_returned_date_sk#15] -Condition : isnotnull(wr_order_number#14) +(20) CometFilter +Input [2]: [wr_order_number#13, wr_returned_date_sk#14] +Condition : isnotnull(wr_order_number#13) -(24) CometProject -Input [2]: [wr_order_number#14, wr_returned_date_sk#15] -Arguments: [wr_order_number#14], [wr_order_number#14] +(21) CometProject +Input [2]: [wr_order_number#13, wr_returned_date_sk#14] +Arguments: [wr_order_number#13], [wr_order_number#13] -(25) CometExchange -Input [1]: [wr_order_number#14] -Arguments: hashpartitioning(wr_order_number#14, 5), Exchange hashpartitioning(wr_order_number#14, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(22) CometExchange +Input [1]: [wr_order_number#13] +Arguments: hashpartitioning(wr_order_number#13, 5), Exchange hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(26) CometSort -Input [1]: [wr_order_number#14] -Arguments: [wr_order_number#14], [wr_order_number#14 ASC NULLS FIRST] +(23) CometSort +Input [1]: [wr_order_number#13] +Arguments: [wr_order_number#13], [wr_order_number#13 ASC NULLS FIRST] -(27) ColumnarToRow [codegen id : 6] -Input [1]: [wr_order_number#14] +(24) ColumnarToRow [codegen id : 6] +Input [1]: [wr_order_number#13] -(28) ReusedExchange [Reuses operator id: 10] +(25) ReusedExchange [Reuses operator id: 10] Output [2]: [ws_warehouse_sk#8, ws_order_number#9] -(29) CometSort +(26) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] -(30) ColumnarToRow [codegen id : 7] +(27) ColumnarToRow [codegen id : 7] Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -(31) ReusedExchange [Reuses operator id: 16] +(28) ReusedExchange [Reuses operator id: 10] Output [2]: [ws_warehouse_sk#11, ws_order_number#12] -(32) CometSort +(29) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 8] +(30) ColumnarToRow [codegen id : 8] Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -(34) SortMergeJoin [codegen id : 9] +(31) SortMergeJoin [codegen id : 9] Left keys [1]: [ws_order_number#9] Right keys [1]: [ws_order_number#12] Join type: Inner Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(35) Project [codegen id : 9] +(32) Project [codegen id : 9] Output [1]: [ws_order_number#9] Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] -(36) SortMergeJoin [codegen id : 10] -Left keys [1]: [wr_order_number#14] +(33) SortMergeJoin [codegen id : 10] +Left keys [1]: [wr_order_number#13] Right keys [1]: [ws_order_number#9] Join type: Inner Join condition: None -(37) Project [codegen id : 10] -Output [1]: [wr_order_number#14] -Input [2]: [wr_order_number#14, ws_order_number#9] +(34) Project [codegen id : 10] +Output [1]: [wr_order_number#13] +Input [2]: [wr_order_number#13, ws_order_number#9] -(38) SortMergeJoin [codegen id : 14] +(35) SortMergeJoin [codegen id : 14] Left keys [1]: [ws_order_number#4] -Right keys [1]: [wr_order_number#14] +Right keys [1]: [wr_order_number#13] Join type: LeftSemi Join condition: None (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#16, d_date#17] +Output [2]: [d_date_sk#15, d_date#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(40) CometFilter -Input [2]: [d_date_sk#16, d_date#17] -Condition : (((isnotnull(d_date#17) AND (d_date#17 >= 1999-02-01)) AND (d_date#17 <= 1999-04-02)) AND isnotnull(d_date_sk#16)) +(37) CometFilter +Input [2]: [d_date_sk#15, d_date#16] +Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 1999-02-01)) AND (d_date#16 <= 1999-04-02)) AND isnotnull(d_date_sk#15)) -(41) CometProject -Input [2]: [d_date_sk#16, d_date#17] -Arguments: [d_date_sk#16], [d_date_sk#16] +(38) CometProject +Input [2]: [d_date_sk#15, d_date#16] +Arguments: [d_date_sk#15], [d_date_sk#15] -(42) ColumnarToRow [codegen id : 11] -Input [1]: [d_date_sk#16] +(39) ColumnarToRow [codegen id : 11] +Input [1]: [d_date_sk#15] -(43) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] +(40) BroadcastExchange +Input [1]: [d_date_sk#15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -(44) BroadcastHashJoin [codegen id : 14] +(41) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#16] +Right keys [1]: [d_date_sk#15] Join type: Inner Join condition: None -(45) Project [codegen id : 14] +(42) Project [codegen id : 14] Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#16] +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#15] (unknown) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_state#19] +Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] ReadSchema: struct -(47) CometFilter -Input [2]: [ca_address_sk#18, ca_state#19] -Condition : ((isnotnull(ca_state#19) AND (ca_state#19 = IL)) AND isnotnull(ca_address_sk#18)) +(44) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : ((isnotnull(ca_state#18) AND (ca_state#18 = IL)) AND isnotnull(ca_address_sk#17)) -(48) CometProject -Input [2]: [ca_address_sk#18, ca_state#19] -Arguments: [ca_address_sk#18], [ca_address_sk#18] +(45) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(49) ColumnarToRow [codegen id : 12] -Input [1]: [ca_address_sk#18] +(46) ColumnarToRow [codegen id : 12] +Input [1]: [ca_address_sk#17] -(50) BroadcastExchange -Input [1]: [ca_address_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +(47) BroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -(51) BroadcastHashJoin [codegen id : 14] +(48) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#18] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(52) Project [codegen id : 14] +(49) Project [codegen id : 14] Output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#18] +Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#17] (unknown) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#20, web_company_name#21] +Output [2]: [web_site_sk#19, web_company_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] ReadSchema: struct -(54) CometFilter -Input [2]: [web_site_sk#20, web_company_name#21] -Condition : ((isnotnull(web_company_name#21) AND (web_company_name#21 = pri )) AND isnotnull(web_site_sk#20)) +(51) CometFilter +Input [2]: [web_site_sk#19, web_company_name#20] +Condition : ((isnotnull(web_company_name#20) AND (web_company_name#20 = pri )) AND isnotnull(web_site_sk#19)) -(55) CometProject -Input [2]: [web_site_sk#20, web_company_name#21] -Arguments: [web_site_sk#20], [web_site_sk#20] +(52) CometProject +Input [2]: [web_site_sk#19, web_company_name#20] +Arguments: [web_site_sk#19], [web_site_sk#19] -(56) ColumnarToRow [codegen id : 13] -Input [1]: [web_site_sk#20] +(53) ColumnarToRow [codegen id : 13] +Input [1]: [web_site_sk#19] -(57) BroadcastExchange -Input [1]: [web_site_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] +(54) BroadcastExchange +Input [1]: [web_site_sk#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] -(58) BroadcastHashJoin [codegen id : 14] +(55) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#20] +Right keys [1]: [web_site_sk#19] Join type: Inner Join condition: None -(59) Project [codegen id : 14] +(56) Project [codegen id : 14] Output [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#20] +Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#19] -(60) HashAggregate [codegen id : 14] +(57) HashAggregate [codegen id : 14] Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Keys [1]: [ws_order_number#4] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23] -Results [3]: [ws_order_number#4, sum#24, sum#25] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] +Results [3]: [ws_order_number#4, sum#23, sum#24] -(61) HashAggregate [codegen id : 14] -Input [3]: [ws_order_number#4, sum#24, sum#25] +(58) HashAggregate [codegen id : 14] +Input [3]: [ws_order_number#4, sum#23, sum#24] Keys [1]: [ws_order_number#4] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23] -Results [3]: [ws_order_number#4, sum#24, sum#25] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] +Results [3]: [ws_order_number#4, sum#23, sum#24] -(62) HashAggregate [codegen id : 14] -Input [3]: [ws_order_number#4, sum#24, sum#25] +(59) HashAggregate [codegen id : 14] +Input [3]: [ws_order_number#4, sum#23, sum#24] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23, count(ws_order_number#4)#26] -Results [3]: [sum#24, sum#25, count#27] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] +Results [3]: [sum#23, sum#24, count#26] -(63) Exchange -Input [3]: [sum#24, sum#25, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(60) Exchange +Input [3]: [sum#23, sum#24, count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] -(64) HashAggregate [codegen id : 15] -Input [3]: [sum#24, sum#25, count#27] +(61) HashAggregate [codegen id : 15] +Input [3]: [sum#23, sum#24, count#26] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#22, sum(UnscaledValue(ws_net_profit#6))#23, count(ws_order_number#4)#26] -Results [3]: [count(ws_order_number#4)#26 AS order count #28, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#22,17,2) AS total shipping cost #29, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#23,17,2) AS total net profit #30] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] +Results [3]: [count(ws_order_number#4)#25 AS order count #27, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#21,17,2) AS total shipping cost #28, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#22,17,2) AS total net profit #29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt index 49ecbf500d..2ad651cb64 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt @@ -43,10 +43,7 @@ WholeStageCodegen (15) ColumnarToRow InputAdapter CometSort [ws_order_number] - CometExchange [ws_order_number] #4 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_order_number,ws_warehouse_sk] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 InputAdapter WholeStageCodegen (10) Project [wr_order_number] @@ -56,7 +53,7 @@ WholeStageCodegen (15) ColumnarToRow InputAdapter CometSort [wr_order_number] - CometExchange [wr_order_number] #5 + CometExchange [wr_order_number] #4 CometProject [wr_order_number] CometFilter [wr_order_number] CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] @@ -75,9 +72,9 @@ WholeStageCodegen (15) ColumnarToRow InputAdapter CometSort [ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #4 + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (11) ColumnarToRow InputAdapter @@ -85,7 +82,7 @@ WholeStageCodegen (15) CometFilter [d_date,d_date_sk] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (12) ColumnarToRow InputAdapter @@ -93,7 +90,7 @@ WholeStageCodegen (15) CometFilter [ca_state,ca_address_sk] CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (13) ColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt index ce84534b1a..a33f704315 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* Sort (192) -+- Exchange (191) - +- * Project (190) - +- * SortMergeJoin Inner (189) +* Sort (179) ++- Exchange (178) + +- * Project (177) + +- * SortMergeJoin Inner (176) :- * Sort (108) : +- Exchange (107) : +- * HashAggregate (106) @@ -111,43 +111,43 @@ : +- CometProject (100) : +- CometFilter (99) : +- CometScan parquet spark_catalog.default.item (98) - +- * Sort (188) - +- Exchange (187) - +- * HashAggregate (186) - +- * HashAggregate (185) - +- * Project (184) - +- * BroadcastHashJoin Inner BuildRight (183) - :- * Project (181) - : +- * BroadcastHashJoin Inner BuildRight (180) - : :- * Project (178) - : : +- * BroadcastHashJoin Inner BuildRight (177) - : : :- * Project (175) - : : : +- * BroadcastHashJoin Inner BuildRight (174) - : : : :- * Project (172) - : : : : +- * BroadcastHashJoin Inner BuildRight (171) - : : : : :- * Project (169) - : : : : : +- * BroadcastHashJoin Inner BuildRight (168) - : : : : : :- * Project (166) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (165) - : : : : : : :- * Project (163) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (162) - : : : : : : : :- * Project (160) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (159) - : : : : : : : : :- * Project (157) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (156) - : : : : : : : : : :- * Project (154) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) - : : : : : : : : : : :- * Project (151) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) - : : : : : : : : : : : :- * Project (148) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) - : : : : : : : : : : : : :- * Project (145) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) - : : : : : : : : : : : : : :- * Project (142) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) - : : : : : : : : : : : : : : :- * ColumnarToRow (139) - : : : : : : : : : : : : : : : +- CometProject (138) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (137) + +- * Sort (175) + +- Exchange (174) + +- * HashAggregate (173) + +- * HashAggregate (172) + +- * Project (171) + +- * BroadcastHashJoin Inner BuildRight (170) + :- * Project (168) + : +- * BroadcastHashJoin Inner BuildRight (167) + : :- * Project (165) + : : +- * BroadcastHashJoin Inner BuildRight (164) + : : :- * Project (162) + : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : :- * Project (159) + : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : :- * Project (156) + : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : :- * Project (153) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : :- * Project (150) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : :- * Project (147) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : :- * Project (144) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : :- * Project (141) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : :- * Project (129) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) + : : : : : : : : : : : : : : :- * ColumnarToRow (126) + : : : : : : : : : : : : : : : +- CometProject (125) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (124) : : : : : : : : : : : : : : : :- CometSort (118) : : : : : : : : : : : : : : : : +- CometExchange (117) : : : : : : : : : : : : : : : : +- CometProject (116) @@ -158,39 +158,26 @@ : : : : : : : : : : : : : : : : +- CometProject (114) : : : : : : : : : : : : : : : : +- CometFilter (113) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (112) - : : : : : : : : : : : : : : : +- CometSort (136) - : : : : : : : : : : : : : : : +- CometProject (135) - : : : : : : : : : : : : : : : +- CometFilter (134) - : : : : : : : : : : : : : : : +- CometHashAggregate (133) - : : : : : : : : : : : : : : : +- CometExchange (132) - : : : : : : : : : : : : : : : +- CometHashAggregate (131) - : : : : : : : : : : : : : : : +- CometProject (130) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (129) - : : : : : : : : : : : : : : : :- CometSort (123) - : : : : : : : : : : : : : : : : +- CometExchange (122) - : : : : : : : : : : : : : : : : +- CometProject (121) - : : : : : : : : : : : : : : : : +- CometFilter (120) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (119) - : : : : : : : : : : : : : : : +- CometSort (128) - : : : : : : : : : : : : : : : +- CometExchange (127) - : : : : : : : : : : : : : : : +- CometProject (126) - : : : : : : : : : : : : : : : +- CometFilter (125) - : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (124) - : : : : : : : : : : : : : : +- ReusedExchange (140) - : : : : : : : : : : : : : +- ReusedExchange (143) - : : : : : : : : : : : : +- ReusedExchange (146) - : : : : : : : : : : : +- ReusedExchange (149) - : : : : : : : : : : +- ReusedExchange (152) - : : : : : : : : : +- ReusedExchange (155) - : : : : : : : : +- ReusedExchange (158) - : : : : : : : +- ReusedExchange (161) - : : : : : : +- ReusedExchange (164) - : : : : : +- ReusedExchange (167) - : : : : +- ReusedExchange (170) - : : : +- ReusedExchange (173) - : : +- ReusedExchange (176) - : +- ReusedExchange (179) - +- ReusedExchange (182) + : : : : : : : : : : : : : : : +- CometSort (123) + : : : : : : : : : : : : : : : +- CometProject (122) + : : : : : : : : : : : : : : : +- CometFilter (121) + : : : : : : : : : : : : : : : +- CometHashAggregate (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : +- ReusedExchange (145) + : : : : : : : +- ReusedExchange (148) + : : : : : : +- ReusedExchange (151) + : : : : : +- ReusedExchange (154) + : : : : +- ReusedExchange (157) + : : : +- ReusedExchange (160) + : : +- ReusedExchange (163) + : +- ReusedExchange (166) + +- ReusedExchange (169) (unknown) Scan parquet spark_catalog.default.store_sales @@ -334,7 +321,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (31) ColumnarToRow [codegen id : 16] Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(32) ReusedExchange [Reuses operator id: 196] +(32) ReusedExchange [Reuses operator id: 183] Output [2]: [d_date_sk#32, d_year#33] (33) BroadcastHashJoin [codegen id : 16] @@ -738,341 +725,280 @@ Arguments: hashpartitioning(ss_item_sk#101, 5), Exchange hashpartitioning(ss_ite Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101 ASC NULLS FIRST] -(unknown) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] -ReadSchema: struct - -(120) CometFilter -Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] -Condition : (isnotnull(cs_item_sk#117) AND isnotnull(cs_order_number#118)) - -(121) CometProject -Input [4]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cs_sold_date_sk#120] -Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] - -(122) CometExchange -Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] -Arguments: hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), Exchange hashpartitioning(cs_item_sk#117, cs_order_number#118, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] - -(123) CometSort -Input [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] -Arguments: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119], [cs_item_sk#117 ASC NULLS FIRST, cs_order_number#118 ASC NULLS FIRST] - -(unknown) Scan parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(125) CometFilter -Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] -Condition : (isnotnull(cr_item_sk#121) AND isnotnull(cr_order_number#122)) - -(126) CometProject -Input [6]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125, cr_returned_date_sk#126] -Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] - -(127) CometExchange -Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), Exchange hashpartitioning(cr_item_sk#121, cr_order_number#122, 5), ENSURE_REQUIREMENTS, [plan_id=23], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=24] +(119) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] -(128) CometSort -Input [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cr_item_sk#121 ASC NULLS FIRST, cr_order_number#122 ASC NULLS FIRST] - -(129) CometSortMergeJoin -Left output [3]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119] -Right output [5]: [cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: [cs_item_sk#117, cs_order_number#118], [cr_item_sk#121, cr_order_number#122], Inner - -(130) CometProject -Input [8]: [cs_item_sk#117, cs_order_number#118, cs_ext_list_price#119, cr_item_sk#121, cr_order_number#122, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] -Arguments: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125], [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] - -(131) CometHashAggregate -Input [5]: [cs_item_sk#117, cs_ext_list_price#119, cr_refunded_cash#123, cr_reversed_charge#124, cr_store_credit#125] +(120) CometHashAggregate +Input [4]: [cs_item_sk#117, sum#118, sum#119, isEmpty#120] Keys [1]: [cs_item_sk#117] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#119)), partial_sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] - -(132) CometExchange -Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] -Arguments: hashpartitioning(cs_item_sk#117, 5), Exchange hashpartitioning(cs_item_sk#117, 5), ENSURE_REQUIREMENTS, [plan_id=25], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=26] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#121)), sum(((cr_refunded_cash#122 + cr_reversed_charge#123) + cr_store_credit#124))] -(133) CometHashAggregate -Input [4]: [cs_item_sk#117, sum#127, sum#128, isEmpty#129] -Keys [1]: [cs_item_sk#117] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#119)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] - -(134) CometFilter +(121) CometFilter Input [3]: [cs_item_sk#117, sale#30, refund#31] Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) -(135) CometProject +(122) CometProject Input [3]: [cs_item_sk#117, sale#30, refund#31] Arguments: [cs_item_sk#117], [cs_item_sk#117] -(136) CometSort +(123) CometSort Input [1]: [cs_item_sk#117] Arguments: [cs_item_sk#117], [cs_item_sk#117 ASC NULLS FIRST] -(137) CometSortMergeJoin +(124) CometSortMergeJoin Left output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] Right output [1]: [cs_item_sk#117] Arguments: [ss_item_sk#101], [cs_item_sk#117], Inner -(138) CometProject +(125) CometProject Input [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, cs_item_sk#117] Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112], [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(139) ColumnarToRow [codegen id : 33] +(126) ColumnarToRow [codegen id : 33] Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -(140) ReusedExchange [Reuses operator id: 200] -Output [2]: [d_date_sk#130, d_year#131] +(127) ReusedExchange [Reuses operator id: 187] +Output [2]: [d_date_sk#125, d_year#126] -(141) BroadcastHashJoin [codegen id : 33] +(128) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_sold_date_sk#112] -Right keys [1]: [d_date_sk#130] +Right keys [1]: [d_date_sk#125] Join type: Inner Join condition: None -(142) Project [codegen id : 33] -Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131] -Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#130, d_year#131] +(129) Project [codegen id : 33] +Output [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126] +Input [13]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112, d_date_sk#125, d_year#126] -(143) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#132, s_store_name#133, s_zip#134] +(130) ReusedExchange [Reuses operator id: 38] +Output [3]: [s_store_sk#127, s_store_name#128, s_zip#129] -(144) BroadcastHashJoin [codegen id : 33] +(131) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_store_sk#106] -Right keys [1]: [s_store_sk#132] +Right keys [1]: [s_store_sk#127] Join type: Inner Join condition: None -(145) Project [codegen id : 33] -Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134] -Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_sk#132, s_store_name#133, s_zip#134] +(132) Project [codegen id : 33] +Output [12]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129] +Input [14]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_sk#127, s_store_name#128, s_zip#129] -(146) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] +(133) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] -(147) BroadcastHashJoin [codegen id : 33] +(134) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_customer_sk#102] -Right keys [1]: [c_customer_sk#135] +Right keys [1]: [c_customer_sk#130] Join type: Inner Join condition: None -(148) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] -Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_customer_sk#135, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140] +(135) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] +Input [18]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_customer_sk#130, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135] -(149) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#141, d_year#142] +(136) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#136, d_year#137] -(150) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_sales_date_sk#140] -Right keys [1]: [d_date_sk#141] +(137) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_sales_date_sk#135] +Right keys [1]: [d_date_sk#136] Join type: Inner Join condition: None -(151) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, c_first_sales_date_sk#140, d_date_sk#141, d_year#142] +(138) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, c_first_sales_date_sk#135, d_date_sk#136, d_year#137] -(152) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#143, d_year#144] +(139) ReusedExchange [Reuses operator id: 50] +Output [2]: [d_date_sk#138, d_year#139] -(153) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_first_shipto_date_sk#139] -Right keys [1]: [d_date_sk#143] +(140) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_first_shipto_date_sk#134] +Right keys [1]: [d_date_sk#138] Join type: Inner Join condition: None -(154) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, c_first_shipto_date_sk#139, d_year#142, d_date_sk#143, d_year#144] +(141) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, c_first_shipto_date_sk#134, d_year#137, d_date_sk#138, d_year#139] -(155) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#145, cd_marital_status#146] +(142) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#140, cd_marital_status#141] -(156) BroadcastHashJoin [codegen id : 33] +(143) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_cdemo_sk#103] -Right keys [1]: [cd_demo_sk#145] +Right keys [1]: [cd_demo_sk#140] Join type: Inner Join condition: None -(157) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146] -Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_demo_sk#145, cd_marital_status#146] +(144) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141] +Input [18]: [ss_item_sk#101, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_demo_sk#140, cd_marital_status#141] -(158) ReusedExchange [Reuses operator id: 59] -Output [2]: [cd_demo_sk#147, cd_marital_status#148] +(145) ReusedExchange [Reuses operator id: 59] +Output [2]: [cd_demo_sk#142, cd_marital_status#143] -(159) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_cdemo_sk#136] -Right keys [1]: [cd_demo_sk#147] +(146) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_cdemo_sk#131] +Right keys [1]: [cd_demo_sk#142] Join type: Inner -Join condition: NOT (cd_marital_status#146 = cd_marital_status#148) +Join condition: NOT (cd_marital_status#141 = cd_marital_status#143) -(160) Project [codegen id : 33] -Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] -Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_cdemo_sk#136, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, cd_marital_status#146, cd_demo_sk#147, cd_marital_status#148] +(147) Project [codegen id : 33] +Output [14]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] +Input [18]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_cdemo_sk#131, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, cd_marital_status#141, cd_demo_sk#142, cd_marital_status#143] -(161) ReusedExchange [Reuses operator id: 68] -Output [1]: [p_promo_sk#149] +(148) ReusedExchange [Reuses operator id: 68] +Output [1]: [p_promo_sk#144] -(162) BroadcastHashJoin [codegen id : 33] +(149) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_promo_sk#107] -Right keys [1]: [p_promo_sk#149] +Right keys [1]: [p_promo_sk#144] Join type: Inner Join condition: None -(163) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, p_promo_sk#149] +(150) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, p_promo_sk#144] -(164) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] +(151) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#145, hd_income_band_sk#146] -(165) BroadcastHashJoin [codegen id : 33] +(152) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_hdemo_sk#104] -Right keys [1]: [hd_demo_sk#150] +Right keys [1]: [hd_demo_sk#145] Join type: Inner Join condition: None -(166) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151] -Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_demo_sk#150, hd_income_band_sk#151] +(153) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146] +Input [15]: [ss_item_sk#101, ss_hdemo_sk#104, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_demo_sk#145, hd_income_band_sk#146] -(167) ReusedExchange [Reuses operator id: 74] -Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] +(154) ReusedExchange [Reuses operator id: 74] +Output [2]: [hd_demo_sk#147, hd_income_band_sk#148] -(168) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_hdemo_sk#137] -Right keys [1]: [hd_demo_sk#152] +(155) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_hdemo_sk#132] +Right keys [1]: [hd_demo_sk#147] Join type: Inner Join condition: None -(169) Project [codegen id : 33] -Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153] -Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_hdemo_sk#137, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_demo_sk#152, hd_income_band_sk#153] +(156) Project [codegen id : 33] +Output [13]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148] +Input [15]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_hdemo_sk#132, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_demo_sk#147, hd_income_band_sk#148] -(170) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(157) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] -(171) BroadcastHashJoin [codegen id : 33] +(158) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_addr_sk#105] -Right keys [1]: [ca_address_sk#154] +Right keys [1]: [ca_address_sk#149] Join type: Inner Join condition: None -(172) Project [codegen id : 33] -Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +(159) Project [codegen id : 33] +Output [16]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] +Input [18]: [ss_item_sk#101, ss_addr_sk#105, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_address_sk#149, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153] -(173) ReusedExchange [Reuses operator id: 83] -Output [5]: [ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +(160) ReusedExchange [Reuses operator id: 83] +Output [5]: [ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(174) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [c_current_addr_sk#138] -Right keys [1]: [ca_address_sk#159] +(161) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [c_current_addr_sk#133] +Right keys [1]: [ca_address_sk#154] Join type: Inner Join condition: None -(175) Project [codegen id : 33] -Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, c_current_addr_sk#138, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_address_sk#159, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] +(162) Project [codegen id : 33] +Output [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [21]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, c_current_addr_sk#133, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_address_sk#154, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] -(176) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#164] +(163) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#159] -(177) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#151] -Right keys [1]: [ib_income_band_sk#164] +(164) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#146] +Right keys [1]: [ib_income_band_sk#159] Join type: Inner Join condition: None -(178) Project [codegen id : 33] -Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#151, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#164] +(165) Project [codegen id : 33] +Output [18]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [20]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#146, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#159] -(179) ReusedExchange [Reuses operator id: 92] -Output [1]: [ib_income_band_sk#165] +(166) ReusedExchange [Reuses operator id: 92] +Output [1]: [ib_income_band_sk#160] -(180) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [hd_income_band_sk#153] -Right keys [1]: [ib_income_band_sk#165] +(167) BroadcastHashJoin [codegen id : 33] +Left keys [1]: [hd_income_band_sk#148] +Right keys [1]: [ib_income_band_sk#160] Join type: Inner Join condition: None -(181) Project [codegen id : 33] -Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, hd_income_band_sk#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, ib_income_band_sk#165] +(168) Project [codegen id : 33] +Output [17]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, hd_income_band_sk#148, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ib_income_band_sk#160] -(182) ReusedExchange [Reuses operator id: 102] -Output [2]: [i_item_sk#166, i_product_name#167] +(169) ReusedExchange [Reuses operator id: 102] +Output [2]: [i_item_sk#161, i_product_name#162] -(183) BroadcastHashJoin [codegen id : 33] +(170) BroadcastHashJoin [codegen id : 33] Left keys [1]: [ss_item_sk#101] -Right keys [1]: [i_item_sk#166] +Right keys [1]: [i_item_sk#161] Join type: Inner Join condition: None -(184) Project [codegen id : 33] -Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] -Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, s_store_name#133, s_zip#134, d_year#142, d_year#144, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] +(171) Project [codegen id : 33] +Output [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] +Input [19]: [ss_item_sk#101, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, s_store_name#128, s_zip#129, d_year#137, d_year#139, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] -(185) HashAggregate [codegen id : 33] -Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#131, d_year#142, d_year#144, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, i_item_sk#166, i_product_name#167] -Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] +(172) HashAggregate [codegen id : 33] +Input [18]: [ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, d_year#126, d_year#137, d_year#139, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, i_item_sk#161, i_product_name#162] +Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#109)), partial_sum(UnscaledValue(ss_list_price#110)), partial_sum(UnscaledValue(ss_coupon_amt#111))] -Aggregate Attributes [4]: [count#72, sum#168, sum#169, sum#170] -Results [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] +Aggregate Attributes [4]: [count#72, sum#163, sum#164, sum#165] +Results [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] -(186) HashAggregate [codegen id : 33] -Input [19]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144, count#76, sum#171, sum#172, sum#173] -Keys [15]: [i_product_name#167, i_item_sk#166, s_store_name#133, s_zip#134, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, ca_street_number#160, ca_street_name#161, ca_city#162, ca_zip#163, d_year#131, d_year#142, d_year#144] +(173) HashAggregate [codegen id : 33] +Input [19]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139, count#76, sum#166, sum#167, sum#168] +Keys [15]: [i_product_name#162, i_item_sk#161, s_store_name#128, s_zip#129, ca_street_number#150, ca_street_name#151, ca_city#152, ca_zip#153, ca_street_number#155, ca_street_name#156, ca_city#157, ca_zip#158, d_year#126, d_year#137, d_year#139] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#109)), sum(UnscaledValue(ss_list_price#110)), sum(UnscaledValue(ss_coupon_amt#111))] Aggregate Attributes [4]: [count(1)#80, sum(UnscaledValue(ss_wholesale_cost#109))#81, sum(UnscaledValue(ss_list_price#110))#82, sum(UnscaledValue(ss_coupon_amt#111))#83] -Results [8]: [i_item_sk#166 AS item_sk#174, s_store_name#133 AS store_name#175, s_zip#134 AS store_zip#176, d_year#131 AS syear#177, count(1)#80 AS cnt#178, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#179, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#180, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#181] +Results [8]: [i_item_sk#161 AS item_sk#169, s_store_name#128 AS store_name#170, s_zip#129 AS store_zip#171, d_year#126 AS syear#172, count(1)#80 AS cnt#173, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#109))#81,17,2) AS s1#174, MakeDecimal(sum(UnscaledValue(ss_list_price#110))#82,17,2) AS s2#175, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#111))#83,17,2) AS s3#176] -(187) Exchange -Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] -Arguments: hashpartitioning(item_sk#174, store_name#175, store_zip#176, 5), ENSURE_REQUIREMENTS, [plan_id=27] +(174) Exchange +Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] +Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=21] -(188) Sort [codegen id : 34] -Input [8]: [item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] -Arguments: [item_sk#174 ASC NULLS FIRST, store_name#175 ASC NULLS FIRST, store_zip#176 ASC NULLS FIRST], false, 0 +(175) Sort [codegen id : 34] +Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] +Arguments: [item_sk#169 ASC NULLS FIRST, store_name#170 ASC NULLS FIRST, store_zip#171 ASC NULLS FIRST], false, 0 -(189) SortMergeJoin [codegen id : 35] +(176) SortMergeJoin [codegen id : 35] Left keys [3]: [item_sk#85, store_name#86, store_zip#87] -Right keys [3]: [item_sk#174, store_name#175, store_zip#176] +Right keys [3]: [item_sk#169, store_name#170, store_zip#171] Join type: Inner -Join condition: (cnt#178 <= cnt#97) +Join condition: (cnt#173 <= cnt#97) -(190) Project [codegen id : 35] -Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] -Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#174, store_name#175, store_zip#176, syear#177, cnt#178, s1#179, s2#180, s3#181] +(177) Project [codegen id : 35] +Output [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] +Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -(191) Exchange -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] -Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#179 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=28] +(178) Exchange +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] +Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#174 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=22] -(192) Sort [codegen id : 36] -Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#179, s2#180, s3#181, syear#177, cnt#178] -Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#178 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#179 ASC NULLS FIRST], true, 0 +(179) Sort [codegen id : 36] +Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] +Arguments: [product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#174 ASC NULLS FIRST], true, 0 ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (196) -+- * ColumnarToRow (195) - +- CometFilter (194) - +- CometScan parquet spark_catalog.default.date_dim (193) +BroadcastExchange (183) ++- * ColumnarToRow (182) + +- CometFilter (181) + +- CometScan parquet spark_catalog.default.date_dim (180) (unknown) Scan parquet spark_catalog.default.date_dim @@ -1082,40 +1008,40 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(194) CometFilter +(181) CometFilter Input [2]: [d_date_sk#32, d_year#33] Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) -(195) ColumnarToRow [codegen id : 1] +(182) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#32, d_year#33] -(196) BroadcastExchange +(183) BroadcastExchange Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=29] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=23] Subquery:2 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#112 IN dynamicpruning#113 -BroadcastExchange (200) -+- * ColumnarToRow (199) - +- CometFilter (198) - +- CometScan parquet spark_catalog.default.date_dim (197) +BroadcastExchange (187) ++- * ColumnarToRow (186) + +- CometFilter (185) + +- CometScan parquet spark_catalog.default.date_dim (184) (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#130, d_year#131] +Output [2]: [d_date_sk#125, d_year#126] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(198) CometFilter -Input [2]: [d_date_sk#130, d_year#131] -Condition : ((isnotnull(d_year#131) AND (d_year#131 = 2000)) AND isnotnull(d_date_sk#130)) +(185) CometFilter +Input [2]: [d_date_sk#125, d_year#126] +Condition : ((isnotnull(d_year#126) AND (d_year#126 = 2000)) AND isnotnull(d_date_sk#125)) -(199) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#130, d_year#131] +(186) ColumnarToRow [codegen id : 1] +Input [2]: [d_date_sk#125, d_year#126] -(200) BroadcastExchange -Input [2]: [d_date_sk#130, d_year#131] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=30] +(187) BroadcastExchange +Input [2]: [d_date_sk#125, d_year#126] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt index 3e9778636a..e2accd7418 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt @@ -221,20 +221,7 @@ WholeStageCodegen (36) CometProject [cs_item_sk] CometFilter [sale,refund] CometHashAggregate [cs_item_sk,sum,sum,isEmpty] - CometExchange [cs_item_sk] #22 - CometHashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number] - CometExchange [cs_item_sk,cs_order_number] #23 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #24 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 InputAdapter ReusedExchange [d_date_sk,d_year] #21 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index 1bf96469f0..c76c310164 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -TakeOrderedAndProject (141) -+- * Project (140) - +- * SortMergeJoin Inner (139) +TakeOrderedAndProject (132) ++- * Project (131) + +- * SortMergeJoin Inner (130) :- * Sort (71) : +- Exchange (70) : +- * Filter (69) @@ -73,18 +73,18 @@ TakeOrderedAndProject (141) : +- CometProject (56) : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.web_returns (54) - +- * Sort (138) - +- Exchange (137) - +- * Filter (136) - +- * HashAggregate (135) - +- Exchange (134) - +- * HashAggregate (133) - +- * HashAggregate (132) - +- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * Project (90) - : +- * SortMergeJoin LeftOuter (89) + +- * Sort (129) + +- Exchange (128) + +- * Filter (127) + +- * HashAggregate (126) + +- Exchange (125) + +- * HashAggregate (124) + +- * HashAggregate (123) + +- Exchange (122) + +- * HashAggregate (121) + +- Union (120) + :- * Project (87) + : +- * SortMergeJoin LeftOuter (86) : :- * Sort (82) : : +- Exchange (81) : : +- * Project (80) @@ -96,50 +96,41 @@ TakeOrderedAndProject (141) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) : : : +- ReusedExchange (75) : : +- ReusedExchange (78) - : +- * ColumnarToRow (88) - : +- CometSort (87) - : +- CometExchange (86) - : +- CometProject (85) - : +- CometFilter (84) - : +- CometScan parquet spark_catalog.default.catalog_returns (83) - :- * Project (109) - : +- * SortMergeJoin LeftOuter (108) - : :- * Sort (101) - : : +- Exchange (100) - : : +- * Project (99) - : : +- * BroadcastHashJoin Inner BuildRight (98) - : : :- * Project (96) - : : : +- * BroadcastHashJoin Inner BuildRight (95) - : : : :- * ColumnarToRow (93) - : : : : +- CometFilter (92) - : : : : +- CometScan parquet spark_catalog.default.store_sales (91) - : : : +- ReusedExchange (94) - : : +- ReusedExchange (97) - : +- * ColumnarToRow (107) - : +- CometSort (106) - : +- CometExchange (105) - : +- CometProject (104) - : +- CometFilter (103) - : +- CometScan parquet spark_catalog.default.store_returns (102) - +- * Project (128) - +- * SortMergeJoin LeftOuter (127) - :- * Sort (120) - : +- Exchange (119) - : +- * Project (118) - : +- * BroadcastHashJoin Inner BuildRight (117) - : :- * Project (115) - : : +- * BroadcastHashJoin Inner BuildRight (114) - : : :- * ColumnarToRow (112) - : : : +- CometFilter (111) - : : : +- CometScan parquet spark_catalog.default.web_sales (110) - : : +- ReusedExchange (113) - : +- ReusedExchange (116) - +- * ColumnarToRow (126) - +- CometSort (125) - +- CometExchange (124) - +- CometProject (123) - +- CometFilter (122) - +- CometScan parquet spark_catalog.default.web_returns (121) + : +- * ColumnarToRow (85) + : +- CometSort (84) + : +- ReusedExchange (83) + :- * Project (103) + : +- * SortMergeJoin LeftOuter (102) + : :- * Sort (98) + : : +- Exchange (97) + : : +- * Project (96) + : : +- * BroadcastHashJoin Inner BuildRight (95) + : : :- * Project (93) + : : : +- * BroadcastHashJoin Inner BuildRight (92) + : : : :- * ColumnarToRow (90) + : : : : +- CometFilter (89) + : : : : +- CometScan parquet spark_catalog.default.store_sales (88) + : : : +- ReusedExchange (91) + : : +- ReusedExchange (94) + : +- * ColumnarToRow (101) + : +- CometSort (100) + : +- ReusedExchange (99) + +- * Project (119) + +- * SortMergeJoin LeftOuter (118) + :- * Sort (114) + : +- Exchange (113) + : +- * Project (112) + : +- * BroadcastHashJoin Inner BuildRight (111) + : :- * Project (109) + : : +- * BroadcastHashJoin Inner BuildRight (108) + : : :- * ColumnarToRow (106) + : : : +- CometFilter (105) + : : : +- CometScan parquet spark_catalog.default.web_sales (104) + : : +- ReusedExchange (107) + : +- ReusedExchange (110) + +- * ColumnarToRow (117) + +- CometSort (116) + +- ReusedExchange (115) (unknown) Scan parquet spark_catalog.default.catalog_sales @@ -189,7 +180,7 @@ Join condition: None Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -(11) ReusedExchange [Reuses operator id: 145] +(11) ReusedExchange [Reuses operator id: 136] Output [2]: [d_date_sk#13, d_year#14] (12) BroadcastHashJoin [codegen id : 3] @@ -274,7 +265,7 @@ Join condition: None Output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -(30) ReusedExchange [Reuses operator id: 145] +(30) ReusedExchange [Reuses operator id: 136] Output [2]: [d_date_sk#33, d_year#34] (31) BroadcastHashJoin [codegen id : 9] @@ -359,7 +350,7 @@ Join condition: None Output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -(49) ReusedExchange [Reuses operator id: 145] +(49) ReusedExchange [Reuses operator id: 136] Output [2]: [d_date_sk#53, d_year#54] (50) BroadcastHashJoin [codegen id : 15] @@ -494,7 +485,7 @@ Join condition: None Output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Input [10]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(78) ReusedExchange [Reuses operator id: 149] +(78) ReusedExchange [Reuses operator id: 140] Output [2]: [d_date_sk#81, d_year#82] (79) BroadcastHashJoin [codegen id : 25] @@ -515,283 +506,235 @@ Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIR Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Arguments: [cs_order_number#71 ASC NULLS FIRST, cs_item_sk#70 ASC NULLS FIRST], false, 0 -(unknown) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(84) CometFilter -Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] -Condition : (isnotnull(cr_order_number#84) AND isnotnull(cr_item_sk#83)) +(83) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(85) CometProject -Input [5]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86, cr_returned_date_sk#87] -Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] - -(86) CometExchange -Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -Arguments: hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), Exchange hashpartitioning(cr_order_number#84, cr_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] - -(87) CometSort +(84) CometSort Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_order_number#84 ASC NULLS FIRST, cr_item_sk#83 ASC NULLS FIRST] -(88) ColumnarToRow [codegen id : 27] +(85) ColumnarToRow [codegen id : 27] Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(89) SortMergeJoin [codegen id : 28] +(86) SortMergeJoin [codegen id : 28] Left keys [2]: [cs_order_number#71, cs_item_sk#70] Right keys [2]: [cr_order_number#84, cr_item_sk#83] Join type: LeftOuter Join condition: None -(90) Project [codegen id : 28] +(87) Project [codegen id : 28] Output [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, (cs_quantity#72 - coalesce(cr_return_quantity#85, 0)) AS sales_cnt#20, (cs_ext_sales_price#73 - coalesce(cr_return_amount#86, 0.00)) AS sales_amt#21] Input [13]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82, cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] (unknown) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] +Output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#92), dynamicpruningexpression(ss_sold_date_sk#92 IN dynamicpruning#93)] +PartitionFilters: [isnotnull(ss_sold_date_sk#91), dynamicpruningexpression(ss_sold_date_sk#91 IN dynamicpruning#92)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(92) CometFilter -Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] -Condition : isnotnull(ss_item_sk#88) +(89) CometFilter +Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +Condition : isnotnull(ss_item_sk#87) -(93) ColumnarToRow [codegen id : 31] -Input [5]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92] +(90) ColumnarToRow [codegen id : 31] +Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -(94) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] +(91) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -(95) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_item_sk#88] -Right keys [1]: [i_item_sk#94] +(92) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_item_sk#87] +Right keys [1]: [i_item_sk#93] Join type: Inner Join condition: None -(96) Project [codegen id : 31] -Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] -Input [10]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_item_sk#94, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98] +(93) Project [codegen id : 31] +Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +Input [10]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -(97) ReusedExchange [Reuses operator id: 149] -Output [2]: [d_date_sk#99, d_year#100] +(94) ReusedExchange [Reuses operator id: 140] +Output [2]: [d_date_sk#98, d_year#99] -(98) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [ss_sold_date_sk#92] -Right keys [1]: [d_date_sk#99] +(95) BroadcastHashJoin [codegen id : 31] +Left keys [1]: [ss_sold_date_sk#91] +Right keys [1]: [d_date_sk#98] Join type: Inner Join condition: None -(99) Project [codegen id : 31] -Output [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] -Input [11]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, ss_sold_date_sk#92, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_date_sk#99, d_year#100] - -(100) Exchange -Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] -Arguments: hashpartitioning(ss_ticket_number#89, ss_item_sk#88, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(101) Sort [codegen id : 32] -Input [9]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100] -Arguments: [ss_ticket_number#89 ASC NULLS FIRST, ss_item_sk#88 ASC NULLS FIRST], false, 0 - -(unknown) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct +(96) Project [codegen id : 31] +Output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_date_sk#98, d_year#99] -(103) CometFilter -Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] -Condition : (isnotnull(sr_ticket_number#102) AND isnotnull(sr_item_sk#101)) +(97) Exchange +Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(104) CometProject -Input [5]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104, sr_returned_date_sk#105] -Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +(98) Sort [codegen id : 32] +Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Arguments: [ss_ticket_number#88 ASC NULLS FIRST, ss_item_sk#87 ASC NULLS FIRST], false, 0 -(105) CometExchange -Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] -Arguments: hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), Exchange hashpartitioning(sr_ticket_number#102, sr_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=18], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=19] +(99) ReusedExchange [Reuses operator id: 38] +Output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -(106) CometSort -Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] -Arguments: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104], [sr_ticket_number#102 ASC NULLS FIRST, sr_item_sk#101 ASC NULLS FIRST] +(100) CometSort +Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +Arguments: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103], [sr_ticket_number#101 ASC NULLS FIRST, sr_item_sk#100 ASC NULLS FIRST] -(107) ColumnarToRow [codegen id : 33] -Input [4]: [sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +(101) ColumnarToRow [codegen id : 33] +Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -(108) SortMergeJoin [codegen id : 34] -Left keys [2]: [ss_ticket_number#89, ss_item_sk#88] -Right keys [2]: [sr_ticket_number#102, sr_item_sk#101] +(102) SortMergeJoin [codegen id : 34] +Left keys [2]: [ss_ticket_number#88, ss_item_sk#87] +Right keys [2]: [sr_ticket_number#101, sr_item_sk#100] Join type: LeftOuter Join condition: None -(109) Project [codegen id : 34] -Output [7]: [d_year#100, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, (ss_quantity#90 - coalesce(sr_return_quantity#103, 0)) AS sales_cnt#40, (ss_ext_sales_price#91 - coalesce(sr_return_amt#104, 0.00)) AS sales_amt#41] -Input [13]: [ss_item_sk#88, ss_ticket_number#89, ss_quantity#90, ss_ext_sales_price#91, i_brand_id#95, i_class_id#96, i_category_id#97, i_manufact_id#98, d_year#100, sr_item_sk#101, sr_ticket_number#102, sr_return_quantity#103, sr_return_amt#104] +(103) Project [codegen id : 34] +Output [7]: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, (ss_quantity#89 - coalesce(sr_return_quantity#102, 0)) AS sales_cnt#40, (ss_ext_sales_price#90 - coalesce(sr_return_amt#103, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99, sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] (unknown) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +Output [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#110), dynamicpruningexpression(ws_sold_date_sk#110 IN dynamicpruning#111)] +PartitionFilters: [isnotnull(ws_sold_date_sk#108), dynamicpruningexpression(ws_sold_date_sk#108 IN dynamicpruning#109)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(111) CometFilter -Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -Condition : isnotnull(ws_item_sk#106) +(105) CometFilter +Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] +Condition : isnotnull(ws_item_sk#104) -(112) ColumnarToRow [codegen id : 37] -Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +(106) ColumnarToRow [codegen id : 37] +Input [5]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108] -(113) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +(107) ReusedExchange [Reuses operator id: 8] +Output [5]: [i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] -(114) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_item_sk#106] -Right keys [1]: [i_item_sk#112] +(108) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_item_sk#104] +Right keys [1]: [i_item_sk#110] Join type: Inner Join condition: None -(115) Project [codegen id : 37] -Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Input [10]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +(109) Project [codegen id : 37] +Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] +Input [10]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_item_sk#110, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114] -(116) ReusedExchange [Reuses operator id: 149] -Output [2]: [d_date_sk#117, d_year#118] +(110) ReusedExchange [Reuses operator id: 140] +Output [2]: [d_date_sk#115, d_year#116] -(117) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_sold_date_sk#110] -Right keys [1]: [d_date_sk#117] +(111) BroadcastHashJoin [codegen id : 37] +Left keys [1]: [ws_sold_date_sk#108] +Right keys [1]: [d_date_sk#115] Join type: Inner Join condition: None -(118) Project [codegen id : 37] -Output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Input [11]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_date_sk#117, d_year#118] - -(119) Exchange -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: hashpartitioning(ws_order_number#107, ws_item_sk#106, 5), ENSURE_REQUIREMENTS, [plan_id=20] - -(120) Sort [codegen id : 38] -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: [ws_order_number#107 ASC NULLS FIRST, ws_item_sk#106 ASC NULLS FIRST], false, 0 - -(unknown) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct +(112) Project [codegen id : 37] +Output [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] +Input [11]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, ws_sold_date_sk#108, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_date_sk#115, d_year#116] -(122) CometFilter -Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] -Condition : (isnotnull(wr_order_number#120) AND isnotnull(wr_item_sk#119)) +(113) Exchange +Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] +Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=16] -(123) CometProject -Input [5]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122, wr_returned_date_sk#123] -Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +(114) Sort [codegen id : 38] +Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] +Arguments: [ws_order_number#105 ASC NULLS FIRST, ws_item_sk#104 ASC NULLS FIRST], false, 0 -(124) CometExchange -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), Exchange hashpartitioning(wr_order_number#120, wr_item_sk#119, 5), ENSURE_REQUIREMENTS, [plan_id=21], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=22] +(115) ReusedExchange [Reuses operator id: 57] +Output [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -(125) CometSort -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_order_number#120 ASC NULLS FIRST, wr_item_sk#119 ASC NULLS FIRST] +(116) CometSort +Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] +Arguments: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120], [wr_order_number#118 ASC NULLS FIRST, wr_item_sk#117 ASC NULLS FIRST] -(126) ColumnarToRow [codegen id : 39] -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +(117) ColumnarToRow [codegen id : 39] +Input [4]: [wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -(127) SortMergeJoin [codegen id : 40] -Left keys [2]: [ws_order_number#107, ws_item_sk#106] -Right keys [2]: [wr_order_number#120, wr_item_sk#119] +(118) SortMergeJoin [codegen id : 40] +Left keys [2]: [ws_order_number#105, ws_item_sk#104] +Right keys [2]: [wr_order_number#118, wr_item_sk#117] Join type: LeftOuter Join condition: None -(128) Project [codegen id : 40] -Output [7]: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, (ws_quantity#108 - coalesce(wr_return_quantity#121, 0)) AS sales_cnt#60, (ws_ext_sales_price#109 - coalesce(wr_return_amt#122, 0.00)) AS sales_amt#61] -Input [13]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118, wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +(119) Project [codegen id : 40] +Output [7]: [d_year#116, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, (ws_quantity#106 - coalesce(wr_return_quantity#119, 0)) AS sales_cnt#60, (ws_ext_sales_price#107 - coalesce(wr_return_amt#120, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116, wr_item_sk#117, wr_order_number#118, wr_return_quantity#119, wr_return_amt#120] -(129) Union +(120) Union -(130) HashAggregate [codegen id : 41] +(121) HashAggregate [codegen id : 41] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(131) Exchange +(122) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=23] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=17] -(132) HashAggregate [codegen id : 42] +(123) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -(133) HashAggregate [codegen id : 42] +(124) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -Aggregate Attributes [2]: [sum#62, sum#124] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] +Aggregate Attributes [2]: [sum#62, sum#121] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] -(134) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=24] +(125) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=18] -(135) HashAggregate [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#125] +(126) HashAggregate [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Aggregate Attributes [2]: [sum(sales_cnt#20)#66, sum(UnscaledValue(sales_amt#21))#67] -Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#126, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#127] +Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#20)#66 AS sales_cnt#123, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#124] -(136) Filter [codegen id : 43] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] -Condition : isnotnull(sales_cnt#126) +(127) Filter [codegen id : 43] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +Condition : isnotnull(sales_cnt#123) -(137) Exchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=25] +(128) Exchange +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=19] -(138) Sort [codegen id : 44] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +(129) Sort [codegen id : 44] +Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] Arguments: [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST], false, 0 -(139) SortMergeJoin [codegen id : 45] +(130) SortMergeJoin [codegen id : 45] Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Right keys [4]: [i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Join type: Inner -Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#126 as decimal(17,2))) < 0.90000000000000000000) +Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#123 as decimal(17,2))) < 0.90000000000000000000) -(140) Project [codegen id : 45] -Output [10]: [d_year#82 AS prev_year#128, d_year#14 AS year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#126 AS prev_yr_cnt#130, sales_cnt#68 AS curr_yr_cnt#131, (sales_cnt#68 - sales_cnt#126) AS sales_cnt_diff#132, (sales_amt#69 - sales_amt#127) AS sales_amt_diff#133] -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#126, sales_amt#127] +(131) Project [codegen id : 45] +Output [10]: [d_year#82 AS prev_year#125, d_year#14 AS year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#123 AS prev_yr_cnt#127, sales_cnt#68 AS curr_yr_cnt#128, (sales_cnt#68 - sales_cnt#123) AS sales_cnt_diff#129, (sales_amt#69 - sales_amt#124) AS sales_amt_diff#130] +Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -(141) TakeOrderedAndProject -Input [10]: [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] -Arguments: 100, [sales_cnt_diff#132 ASC NULLS FIRST, sales_amt_diff#133 ASC NULLS FIRST], [prev_year#128, year#129, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#130, curr_yr_cnt#131, sales_cnt_diff#132, sales_amt_diff#133] +(132) TakeOrderedAndProject +Input [10]: [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] +Arguments: 100, [sales_cnt_diff#129 ASC NULLS FIRST, sales_amt_diff#130 ASC NULLS FIRST], [prev_year#125, year#126, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#127, curr_yr_cnt#128, sales_cnt_diff#129, sales_amt_diff#130] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (145) -+- * ColumnarToRow (144) - +- CometFilter (143) - +- CometScan parquet spark_catalog.default.date_dim (142) +BroadcastExchange (136) ++- * ColumnarToRow (135) + +- CometFilter (134) + +- CometScan parquet spark_catalog.default.date_dim (133) (unknown) Scan parquet spark_catalog.default.date_dim @@ -801,26 +744,26 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(143) CometFilter +(134) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(144) ColumnarToRow [codegen id : 1] +(135) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -(145) BroadcastExchange +(136) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=26] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=20] Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 -BroadcastExchange (149) -+- * ColumnarToRow (148) - +- CometFilter (147) - +- CometScan parquet spark_catalog.default.date_dim (146) +BroadcastExchange (140) ++- * ColumnarToRow (139) + +- CometFilter (138) + +- CometScan parquet spark_catalog.default.date_dim (137) (unknown) Scan parquet spark_catalog.default.date_dim @@ -830,19 +773,19 @@ Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(147) CometFilter +(138) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(148) ColumnarToRow [codegen id : 1] +(139) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -(149) BroadcastExchange +(140) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=27] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=21] -Subquery:5 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#92 IN dynamicpruning#75 +Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#75 -Subquery:6 Hosting operator id = 110 Hosting Expression = ws_sold_date_sk#110 IN dynamicpruning#75 +Subquery:6 Hosting operator id = 104 Hosting Expression = ws_sold_date_sk#108 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt index 1dc6c29cba..25dd0f9468 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt @@ -178,10 +178,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [cr_order_number,cr_item_sk] - CometExchange [cr_order_number,cr_item_sk] #17 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_order_number,cr_item_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 WholeStageCodegen (34) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] @@ -189,7 +186,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i WholeStageCodegen (32) Sort [ss_ticket_number,ss_item_sk] InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #18 + Exchange [ss_ticket_number,ss_item_sk] #17 WholeStageCodegen (31) Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ss_sold_date_sk,d_date_sk] @@ -209,10 +206,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [sr_ticket_number,sr_item_sk] - CometExchange [sr_ticket_number,sr_item_sk] #19 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_ticket_number,sr_item_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 WholeStageCodegen (40) Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] @@ -220,7 +214,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i WholeStageCodegen (38) Sort [ws_order_number,ws_item_sk] InputAdapter - Exchange [ws_order_number,ws_item_sk] #20 + Exchange [ws_order_number,ws_item_sk] #18 WholeStageCodegen (37) Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] BroadcastHashJoin [ws_sold_date_sk,d_date_sk] @@ -240,7 +234,4 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [wr_order_number,wr_item_sk] - CometExchange [wr_order_number,wr_item_sk] #21 - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_order_number,wr_item_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt index 334eab7c7b..d9d9be3c1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -TakeOrderedAndProject (265) -+- * HashAggregate (264) - +- Exchange (263) - +- * HashAggregate (262) - +- Union (261) +TakeOrderedAndProject (117) ++- * HashAggregate (116) + +- Exchange (115) + +- * HashAggregate (114) + +- Union (113) :- * HashAggregate (102) : +- Exchange (101) : +- * HashAggregate (100) @@ -106,164 +106,16 @@ TakeOrderedAndProject (265) : : : +- CometScan parquet spark_catalog.default.web_site (84) : : +- ReusedExchange (90) : +- ReusedExchange (93) - :- * HashAggregate (181) - : +- Exchange (180) - : +- * HashAggregate (179) - : +- * HashAggregate (178) - : +- Exchange (177) - : +- * HashAggregate (176) - : +- Union (175) - : :- * HashAggregate (126) - : : +- Exchange (125) - : : +- * HashAggregate (124) - : : +- * Project (123) - : : +- * BroadcastHashJoin Inner BuildRight (122) - : : :- * Project (120) - : : : +- * BroadcastHashJoin Inner BuildRight (119) - : : : :- * Project (117) - : : : : +- * BroadcastHashJoin Inner BuildRight (116) - : : : : :- * Project (114) - : : : : : +- * BroadcastHashJoin Inner BuildRight (113) - : : : : : :- * ColumnarToRow (111) - : : : : : : +- CometProject (110) - : : : : : : +- CometSortMergeJoin (109) - : : : : : : :- CometSort (106) - : : : : : : : +- CometExchange (105) - : : : : : : : +- CometFilter (104) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (103) - : : : : : : +- CometSort (108) - : : : : : : +- ReusedExchange (107) - : : : : : +- ReusedExchange (112) - : : : : +- ReusedExchange (115) - : : : +- ReusedExchange (118) - : : +- ReusedExchange (121) - : :- * HashAggregate (150) - : : +- Exchange (149) - : : +- * HashAggregate (148) - : : +- * Project (147) - : : +- * BroadcastHashJoin Inner BuildRight (146) - : : :- * Project (144) - : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : :- * Project (141) - : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : :- * Project (138) - : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : :- * ColumnarToRow (135) - : : : : : : +- CometProject (134) - : : : : : : +- CometSortMergeJoin (133) - : : : : : : :- CometSort (130) - : : : : : : : +- CometExchange (129) - : : : : : : : +- CometFilter (128) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (127) - : : : : : : +- CometSort (132) - : : : : : : +- ReusedExchange (131) - : : : : : +- ReusedExchange (136) - : : : : +- ReusedExchange (139) - : : : +- ReusedExchange (142) - : : +- ReusedExchange (145) - : +- * HashAggregate (174) - : +- Exchange (173) - : +- * HashAggregate (172) - : +- * Project (171) - : +- * BroadcastHashJoin Inner BuildRight (170) - : :- * Project (168) - : : +- * BroadcastHashJoin Inner BuildRight (167) - : : :- * Project (165) - : : : +- * BroadcastHashJoin Inner BuildRight (164) - : : : :- * Project (162) - : : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : : :- * ColumnarToRow (159) - : : : : : +- CometProject (158) - : : : : : +- CometSortMergeJoin (157) - : : : : : :- CometSort (154) - : : : : : : +- CometExchange (153) - : : : : : : +- CometFilter (152) - : : : : : : +- CometScan parquet spark_catalog.default.web_sales (151) - : : : : : +- CometSort (156) - : : : : : +- ReusedExchange (155) - : : : : +- ReusedExchange (160) - : : : +- ReusedExchange (163) - : : +- ReusedExchange (166) - : +- ReusedExchange (169) - +- * HashAggregate (260) - +- Exchange (259) - +- * HashAggregate (258) - +- * HashAggregate (257) - +- Exchange (256) - +- * HashAggregate (255) - +- Union (254) - :- * HashAggregate (205) - : +- Exchange (204) - : +- * HashAggregate (203) - : +- * Project (202) - : +- * BroadcastHashJoin Inner BuildRight (201) - : :- * Project (199) - : : +- * BroadcastHashJoin Inner BuildRight (198) - : : :- * Project (196) - : : : +- * BroadcastHashJoin Inner BuildRight (195) - : : : :- * Project (193) - : : : : +- * BroadcastHashJoin Inner BuildRight (192) - : : : : :- * ColumnarToRow (190) - : : : : : +- CometProject (189) - : : : : : +- CometSortMergeJoin (188) - : : : : : :- CometSort (185) - : : : : : : +- CometExchange (184) - : : : : : : +- CometFilter (183) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (182) - : : : : : +- CometSort (187) - : : : : : +- ReusedExchange (186) - : : : : +- ReusedExchange (191) - : : : +- ReusedExchange (194) - : : +- ReusedExchange (197) - : +- ReusedExchange (200) - :- * HashAggregate (229) - : +- Exchange (228) - : +- * HashAggregate (227) - : +- * Project (226) - : +- * BroadcastHashJoin Inner BuildRight (225) - : :- * Project (223) - : : +- * BroadcastHashJoin Inner BuildRight (222) - : : :- * Project (220) - : : : +- * BroadcastHashJoin Inner BuildRight (219) - : : : :- * Project (217) - : : : : +- * BroadcastHashJoin Inner BuildRight (216) - : : : : :- * ColumnarToRow (214) - : : : : : +- CometProject (213) - : : : : : +- CometSortMergeJoin (212) - : : : : : :- CometSort (209) - : : : : : : +- CometExchange (208) - : : : : : : +- CometFilter (207) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (206) - : : : : : +- CometSort (211) - : : : : : +- ReusedExchange (210) - : : : : +- ReusedExchange (215) - : : : +- ReusedExchange (218) - : : +- ReusedExchange (221) - : +- ReusedExchange (224) - +- * HashAggregate (253) - +- Exchange (252) - +- * HashAggregate (251) - +- * Project (250) - +- * BroadcastHashJoin Inner BuildRight (249) - :- * Project (247) - : +- * BroadcastHashJoin Inner BuildRight (246) - : :- * Project (244) - : : +- * BroadcastHashJoin Inner BuildRight (243) - : : :- * Project (241) - : : : +- * BroadcastHashJoin Inner BuildRight (240) - : : : :- * ColumnarToRow (238) - : : : : +- CometProject (237) - : : : : +- CometSortMergeJoin (236) - : : : : :- CometSort (233) - : : : : : +- CometExchange (232) - : : : : : +- CometFilter (231) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (230) - : : : : +- CometSort (235) - : : : : +- ReusedExchange (234) - : : : +- ReusedExchange (239) - : : +- ReusedExchange (242) - : +- ReusedExchange (245) - +- ReusedExchange (248) + :- * HashAggregate (107) + : +- Exchange (106) + : +- * HashAggregate (105) + : +- * HashAggregate (104) + : +- ReusedExchange (103) + +- * HashAggregate (112) + +- Exchange (111) + +- * HashAggregate (110) + +- * HashAggregate (109) + +- ReusedExchange (108) (unknown) Scan parquet spark_catalog.default.store_sales @@ -321,7 +173,7 @@ Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss (12) ColumnarToRow [codegen id : 5] Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -(13) ReusedExchange [Reuses operator id: 270] +(13) ReusedExchange [Reuses operator id: 122] Output [1]: [d_date_sk#14] (14) BroadcastHashJoin [codegen id : 5] @@ -499,7 +351,7 @@ Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_p (50) ColumnarToRow [codegen id : 11] Input [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] -(51) ReusedExchange [Reuses operator id: 270] +(51) ReusedExchange [Reuses operator id: 122] Output [1]: [d_date_sk#52] (52) BroadcastHashJoin [codegen id : 11] @@ -639,7 +491,7 @@ Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price (80) ColumnarToRow [codegen id : 17] Input [8]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] -(81) ReusedExchange [Reuses operator id: 270] +(81) ReusedExchange [Reuses operator id: 122] Output [1]: [d_date_sk#88] (82) BroadcastHashJoin [codegen id : 17] @@ -744,806 +596,120 @@ Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#123, sum(returns#37)#124, sum(profit#38)#125] Results [5]: [channel#34, id#35, cast(sum(sales#36)#123 as decimal(37,2)) AS sales#126, cast(sum(returns#37)#124 as decimal(38,2)) AS returns#127, cast(sum(profit#38)#125 as decimal(38,2)) AS profit#128] -(unknown) Scan parquet spark_catalog.default.store_sales -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#129)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] -ReadSchema: struct - -(104) CometFilter -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(105) CometExchange -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=22], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=23] - -(106) CometSort -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] - -(107) ReusedExchange [Reuses operator id: 8] -Output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] - -(108) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] - -(109) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter - -(110) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] - -(111) ColumnarToRow [codegen id : 25] -Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] - -(112) ReusedExchange [Reuses operator id: 270] -Output [1]: [d_date_sk#14] - -(113) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(114) Project [codegen id : 25] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] - -(115) ReusedExchange [Reuses operator id: 19] -Output [2]: [s_store_sk#15, s_store_id#16] - -(116) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(117) Project [codegen id : 25] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#15, s_store_id#16] - -(118) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#17] - -(119) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(120) Project [codegen id : 25] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, i_item_sk#17] - -(121) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#19] - -(122) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 25] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, p_promo_sk#19] - -(124) HashAggregate [codegen id : 25] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Keys [1]: [s_store_id#16] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#130, sum#131, isEmpty#132, sum#133, isEmpty#134] -Results [6]: [s_store_id#16, sum#135, sum#136, isEmpty#137, sum#138, isEmpty#139] +(103) ReusedExchange [Reuses operator id: 101] +Output [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] -(125) Exchange -Input [6]: [s_store_id#16, sum#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=24] - -(126) HashAggregate [codegen id : 26] -Input [6]: [s_store_id#16, sum#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33] -Results [5]: [store channel AS channel#34, concat(store, s_store_id#16) AS id#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33 AS profit#38] - -(unknown) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_sold_date_sk#45 IN dynamicpruning#140)] -PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] -ReadSchema: struct - -(128) CometFilter -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) - -(129) CometExchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=25], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=26] - -(130) CometSort -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45], [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST] - -(131) ReusedExchange [Reuses operator id: 46] -Output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] - -(132) CometSort -Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50], [cr_item_sk#47 ASC NULLS FIRST, cr_order_number#48 ASC NULLS FIRST] - -(133) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Right output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: [cs_item_sk#40, cs_order_number#42], [cr_item_sk#47, cr_order_number#48], LeftOuter - -(134) CometProject -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] - -(135) ColumnarToRow [codegen id : 31] -Input [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] - -(136) ReusedExchange [Reuses operator id: 270] -Output [1]: [d_date_sk#52] - -(137) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#52] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 31] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50, d_date_sk#52] - -(139) ReusedExchange [Reuses operator id: 57] -Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] - -(140) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#53] -Join type: Inner -Join condition: None - -(141) Project [codegen id : 31] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_sk#53, cp_catalog_page_id#54] - -(142) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#55] - -(143) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#55] -Join type: Inner -Join condition: None - -(144) Project [codegen id : 31] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, i_item_sk#55] - -(145) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#56] - -(146) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#56] -Join type: Inner -Join condition: None - -(147) Project [codegen id : 31] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, p_promo_sk#56] - -(148) HashAggregate [codegen id : 31] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Keys [1]: [cp_catalog_page_id#54] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Results [6]: [cp_catalog_page_id#54, sum#146, sum#147, isEmpty#148, sum#149, isEmpty#150] - -(149) Exchange -Input [6]: [cp_catalog_page_id#54, sum#146, sum#147, isEmpty#148, sum#149, isEmpty#150] -Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=27] - -(150) HashAggregate [codegen id : 32] -Input [6]: [cp_catalog_page_id#54, sum#146, sum#147, isEmpty#148, sum#149, isEmpty#150] -Keys [1]: [cp_catalog_page_id#54] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#67, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69] -Results [5]: [catalog channel AS channel#70, concat(catalog_page, cp_catalog_page_id#54) AS id#71, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#67,17,2) AS sales#72, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68 AS returns#73, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69 AS profit#74] - -(unknown) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#151)] -PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] -ReadSchema: struct - -(152) CometFilter -Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) - -(153) CometExchange -Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=28], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=29] - -(154) CometSort -Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81], [ws_item_sk#75 ASC NULLS FIRST, ws_order_number#78 ASC NULLS FIRST] - -(155) ReusedExchange [Reuses operator id: 76] -Output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] - -(156) CometSort -Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86], [wr_item_sk#83 ASC NULLS FIRST, wr_order_number#84 ASC NULLS FIRST] - -(157) CometSortMergeJoin -Left output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Right output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: [ws_item_sk#75, ws_order_number#78], [wr_item_sk#83, wr_order_number#84], LeftOuter - -(158) CometProject -Input [11]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86], [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] - -(159) ColumnarToRow [codegen id : 37] -Input [8]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] - -(160) ReusedExchange [Reuses operator id: 270] -Output [1]: [d_date_sk#88] - -(161) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_sold_date_sk#81] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None - -(162) Project [codegen id : 37] -Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86] -Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86, d_date_sk#88] - -(163) ReusedExchange [Reuses operator id: 87] -Output [2]: [web_site_sk#89, web_site_id#90] - -(164) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_web_site_sk#76] -Right keys [1]: [web_site_sk#89] -Join type: Inner -Join condition: None - -(165) Project [codegen id : 37] -Output [7]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_sk#89, web_site_id#90] - -(166) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#91] - -(167) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [i_item_sk#91] -Join type: Inner -Join condition: None - -(168) Project [codegen id : 37] -Output [6]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Input [8]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, i_item_sk#91] - -(169) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#92] - -(170) BroadcastHashJoin [codegen id : 37] -Left keys [1]: [ws_promo_sk#77] -Right keys [1]: [p_promo_sk#92] -Join type: Inner -Join condition: None - -(171) Project [codegen id : 37] -Output [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Input [7]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, p_promo_sk#92] - -(172) HashAggregate [codegen id : 37] -Input [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Keys [1]: [web_site_id#90] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#79)), partial_sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#152, sum#153, isEmpty#154, sum#155, isEmpty#156] -Results [6]: [web_site_id#90, sum#157, sum#158, isEmpty#159, sum#160, isEmpty#161] - -(173) Exchange -Input [6]: [web_site_id#90, sum#157, sum#158, isEmpty#159, sum#160, isEmpty#161] -Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=30] - -(174) HashAggregate [codegen id : 38] -Input [6]: [web_site_id#90, sum#157, sum#158, isEmpty#159, sum#160, isEmpty#161] -Keys [1]: [web_site_id#90] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#79)), sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#79))#103, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105] -Results [5]: [web channel AS channel#106, concat(web_site, web_site_id#90) AS id#107, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#79))#103,17,2) AS sales#108, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104 AS returns#109, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105 AS profit#110] - -(175) Union - -(176) HashAggregate [codegen id : 39] -Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] -Keys [2]: [channel#34, id#35] -Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] -Aggregate Attributes [6]: [sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116] -Results [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] - -(177) Exchange -Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=31] - -(178) HashAggregate [codegen id : 40] +(104) HashAggregate [codegen id : 40] Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#123, sum(returns#37)#124, sum(profit#38)#125] -Results [4]: [channel#34, sum(sales#36)#123 AS sales#162, sum(returns#37)#124 AS returns#163, sum(profit#38)#125 AS profit#164] +Results [4]: [channel#34, sum(sales#36)#123 AS sales#129, sum(returns#37)#124 AS returns#130, sum(profit#38)#125 AS profit#131] -(179) HashAggregate [codegen id : 40] -Input [4]: [channel#34, sales#162, returns#163, profit#164] +(105) HashAggregate [codegen id : 40] +Input [4]: [channel#34, sales#129, returns#130, profit#131] Keys [1]: [channel#34] -Functions [3]: [partial_sum(sales#162), partial_sum(returns#163), partial_sum(profit#164)] -Aggregate Attributes [6]: [sum#165, isEmpty#166, sum#167, isEmpty#168, sum#169, isEmpty#170] -Results [7]: [channel#34, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Functions [3]: [partial_sum(sales#129), partial_sum(returns#130), partial_sum(profit#131)] +Aggregate Attributes [6]: [sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Results [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -(180) Exchange -Input [7]: [channel#34, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=32] +(106) Exchange +Input [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=22] -(181) HashAggregate [codegen id : 41] -Input [7]: [channel#34, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +(107) HashAggregate [codegen id : 41] +Input [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] Keys [1]: [channel#34] -Functions [3]: [sum(sales#162), sum(returns#163), sum(profit#164)] -Aggregate Attributes [3]: [sum(sales#162)#177, sum(returns#163)#178, sum(profit#164)#179] -Results [5]: [channel#34, null AS id#180, sum(sales#162)#177 AS sales#181, sum(returns#163)#178 AS returns#182, sum(profit#164)#179 AS profit#183] - -(unknown) Scan parquet spark_catalog.default.store_sales -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#184)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] -ReadSchema: struct - -(183) CometFilter -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(184) CometExchange -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=33], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=34] - -(185) CometSort -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] - -(186) ReusedExchange [Reuses operator id: 8] -Output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] - -(187) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] - -(188) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter - -(189) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] - -(190) ColumnarToRow [codegen id : 46] -Input [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] - -(191) ReusedExchange [Reuses operator id: 270] -Output [1]: [d_date_sk#14] - -(192) BroadcastHashJoin [codegen id : 46] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(193) Project [codegen id : 46] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] - -(194) ReusedExchange [Reuses operator id: 19] -Output [2]: [s_store_sk#15, s_store_id#16] - -(195) BroadcastHashJoin [codegen id : 46] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(196) Project [codegen id : 46] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#15, s_store_id#16] - -(197) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#17] - -(198) BroadcastHashJoin [codegen id : 46] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(199) Project [codegen id : 46] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, i_item_sk#17] - -(200) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#19] - -(201) BroadcastHashJoin [codegen id : 46] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None - -(202) Project [codegen id : 46] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16, p_promo_sk#19] - -(203) HashAggregate [codegen id : 46] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#16] -Keys [1]: [s_store_id#16] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#185, sum#186, isEmpty#187, sum#188, isEmpty#189] -Results [6]: [s_store_id#16, sum#190, sum#191, isEmpty#192, sum#193, isEmpty#194] - -(204) Exchange -Input [6]: [s_store_id#16, sum#190, sum#191, isEmpty#192, sum#193, isEmpty#194] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=35] - -(205) HashAggregate [codegen id : 47] -Input [6]: [s_store_id#16, sum#190, sum#191, isEmpty#192, sum#193, isEmpty#194] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33] -Results [5]: [store channel AS channel#34, concat(store, s_store_id#16) AS id#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#32 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#33 AS profit#38] - -(unknown) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45), dynamicpruningexpression(cs_sold_date_sk#45 IN dynamicpruning#195)] -PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] -ReadSchema: struct - -(207) CometFilter -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) - -(208) CometExchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=36], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=37] - -(209) CometSort -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45], [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST] - -(210) ReusedExchange [Reuses operator id: 46] -Output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] - -(211) CometSort -Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50], [cr_item_sk#47 ASC NULLS FIRST, cr_order_number#48 ASC NULLS FIRST] +Functions [3]: [sum(sales#129), sum(returns#130), sum(profit#131)] +Aggregate Attributes [3]: [sum(sales#129)#144, sum(returns#130)#145, sum(profit#131)#146] +Results [5]: [channel#34, null AS id#147, sum(sales#129)#144 AS sales#148, sum(returns#130)#145 AS returns#149, sum(profit#131)#146 AS profit#150] -(212) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Right output [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: [cs_item_sk#40, cs_order_number#42], [cr_item_sk#47, cr_order_number#48], LeftOuter - -(213) CometProject -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] - -(214) ColumnarToRow [codegen id : 52] -Input [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50] - -(215) ReusedExchange [Reuses operator id: 270] -Output [1]: [d_date_sk#52] - -(216) BroadcastHashJoin [codegen id : 52] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#52] -Join type: Inner -Join condition: None - -(217) Project [codegen id : 52] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#49, cr_net_loss#50, d_date_sk#52] - -(218) ReusedExchange [Reuses operator id: 57] -Output [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] - -(219) BroadcastHashJoin [codegen id : 52] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#53] -Join type: Inner -Join condition: None - -(220) Project [codegen id : 52] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_sk#53, cp_catalog_page_id#54] - -(221) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#55] +(108) ReusedExchange [Reuses operator id: 101] +Output [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] -(222) BroadcastHashJoin [codegen id : 52] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#55] -Join type: Inner -Join condition: None - -(223) Project [codegen id : 52] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, i_item_sk#55] - -(224) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#56] - -(225) BroadcastHashJoin [codegen id : 52] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#56] -Join type: Inner -Join condition: None - -(226) Project [codegen id : 52] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54, p_promo_sk#56] - -(227) HashAggregate [codegen id : 52] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#49, cr_net_loss#50, cp_catalog_page_id#54] -Keys [1]: [cp_catalog_page_id#54] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#196, sum#197, isEmpty#198, sum#199, isEmpty#200] -Results [6]: [cp_catalog_page_id#54, sum#201, sum#202, isEmpty#203, sum#204, isEmpty#205] - -(228) Exchange -Input [6]: [cp_catalog_page_id#54, sum#201, sum#202, isEmpty#203, sum#204, isEmpty#205] -Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=38] - -(229) HashAggregate [codegen id : 53] -Input [6]: [cp_catalog_page_id#54, sum#201, sum#202, isEmpty#203, sum#204, isEmpty#205] -Keys [1]: [cp_catalog_page_id#54] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#67, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69] -Results [5]: [catalog channel AS channel#70, concat(catalog_page, cp_catalog_page_id#54) AS id#71, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#67,17,2) AS sales#72, sum(coalesce(cast(cr_return_amount#49 as decimal(12,2)), 0.00))#68 AS returns#73, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#50 as decimal(12,2)), 0.00)))#69 AS profit#74] - -(unknown) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#206)] -PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] -ReadSchema: struct - -(231) CometFilter -Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isnotnull(ws_promo_sk#77)) - -(232) CometExchange -Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=39], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=40] - -(233) CometSort -Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81], [ws_item_sk#75 ASC NULLS FIRST, ws_order_number#78 ASC NULLS FIRST] - -(234) ReusedExchange [Reuses operator id: 76] -Output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] - -(235) CometSort -Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86], [wr_item_sk#83 ASC NULLS FIRST, wr_order_number#84 ASC NULLS FIRST] - -(236) CometSortMergeJoin -Left output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Right output [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: [ws_item_sk#75, ws_order_number#78], [wr_item_sk#83, wr_order_number#84], LeftOuter - -(237) CometProject -Input [11]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86], [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] - -(238) ColumnarToRow [codegen id : 58] -Input [8]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86] - -(239) ReusedExchange [Reuses operator id: 270] -Output [1]: [d_date_sk#88] - -(240) BroadcastHashJoin [codegen id : 58] -Left keys [1]: [ws_sold_date_sk#81] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None - -(241) Project [codegen id : 58] -Output [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86] -Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81, wr_return_amt#85, wr_net_loss#86, d_date_sk#88] - -(242) ReusedExchange [Reuses operator id: 87] -Output [2]: [web_site_sk#89, web_site_id#90] - -(243) BroadcastHashJoin [codegen id : 58] -Left keys [1]: [ws_web_site_sk#76] -Right keys [1]: [web_site_sk#89] -Join type: Inner -Join condition: None - -(244) Project [codegen id : 58] -Output [7]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Input [9]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_sk#89, web_site_id#90] - -(245) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#91] - -(246) BroadcastHashJoin [codegen id : 58] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [i_item_sk#91] -Join type: Inner -Join condition: None - -(247) Project [codegen id : 58] -Output [6]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Input [8]: [ws_item_sk#75, ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, i_item_sk#91] - -(248) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#92] - -(249) BroadcastHashJoin [codegen id : 58] -Left keys [1]: [ws_promo_sk#77] -Right keys [1]: [p_promo_sk#92] -Join type: Inner -Join condition: None - -(250) Project [codegen id : 58] -Output [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Input [7]: [ws_promo_sk#77, ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90, p_promo_sk#92] - -(251) HashAggregate [codegen id : 58] -Input [5]: [ws_ext_sales_price#79, ws_net_profit#80, wr_return_amt#85, wr_net_loss#86, web_site_id#90] -Keys [1]: [web_site_id#90] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#79)), partial_sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#207, sum#208, isEmpty#209, sum#210, isEmpty#211] -Results [6]: [web_site_id#90, sum#212, sum#213, isEmpty#214, sum#215, isEmpty#216] - -(252) Exchange -Input [6]: [web_site_id#90, sum#212, sum#213, isEmpty#214, sum#215, isEmpty#216] -Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=41] - -(253) HashAggregate [codegen id : 59] -Input [6]: [web_site_id#90, sum#212, sum#213, isEmpty#214, sum#215, isEmpty#216] -Keys [1]: [web_site_id#90] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#79)), sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00)), sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#79))#103, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105] -Results [5]: [web channel AS channel#106, concat(web_site, web_site_id#90) AS id#107, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#79))#103,17,2) AS sales#108, sum(coalesce(cast(wr_return_amt#85 as decimal(12,2)), 0.00))#104 AS returns#109, sum((ws_net_profit#80 - coalesce(cast(wr_net_loss#86 as decimal(12,2)), 0.00)))#105 AS profit#110] - -(254) Union - -(255) HashAggregate [codegen id : 60] -Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] -Keys [2]: [channel#34, id#35] -Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] -Aggregate Attributes [6]: [sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116] -Results [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] - -(256) Exchange -Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=42] - -(257) HashAggregate [codegen id : 61] +(109) HashAggregate [codegen id : 61] Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#123, sum(returns#37)#124, sum(profit#38)#125] -Results [3]: [sum(sales#36)#123 AS sales#162, sum(returns#37)#124 AS returns#163, sum(profit#38)#125 AS profit#164] +Results [3]: [sum(sales#36)#123 AS sales#129, sum(returns#37)#124 AS returns#130, sum(profit#38)#125 AS profit#131] -(258) HashAggregate [codegen id : 61] -Input [3]: [sales#162, returns#163, profit#164] +(110) HashAggregate [codegen id : 61] +Input [3]: [sales#129, returns#130, profit#131] Keys: [] -Functions [3]: [partial_sum(sales#162), partial_sum(returns#163), partial_sum(profit#164)] -Aggregate Attributes [6]: [sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222] -Results [6]: [sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228] +Functions [3]: [partial_sum(sales#129), partial_sum(returns#130), partial_sum(profit#131)] +Aggregate Attributes [6]: [sum#151, isEmpty#152, sum#153, isEmpty#154, sum#155, isEmpty#156] +Results [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] -(259) Exchange -Input [6]: [sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=43] +(111) Exchange +Input [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=23] -(260) HashAggregate [codegen id : 62] -Input [6]: [sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228] +(112) HashAggregate [codegen id : 62] +Input [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] Keys: [] -Functions [3]: [sum(sales#162), sum(returns#163), sum(profit#164)] -Aggregate Attributes [3]: [sum(sales#162)#229, sum(returns#163)#230, sum(profit#164)#231] -Results [5]: [null AS channel#232, null AS id#233, sum(sales#162)#229 AS sales#234, sum(returns#163)#230 AS returns#235, sum(profit#164)#231 AS profit#236] +Functions [3]: [sum(sales#129), sum(returns#130), sum(profit#131)] +Aggregate Attributes [3]: [sum(sales#129)#163, sum(returns#130)#164, sum(profit#131)#165] +Results [5]: [null AS channel#166, null AS id#167, sum(sales#129)#163 AS sales#168, sum(returns#130)#164 AS returns#169, sum(profit#131)#165 AS profit#170] -(261) Union +(113) Union -(262) HashAggregate [codegen id : 63] +(114) HashAggregate [codegen id : 63] Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Keys [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Functions: [] Aggregate Attributes: [] Results [5]: [channel#34, id#35, sales#126, returns#127, profit#128] -(263) Exchange +(115) Exchange Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] -Arguments: hashpartitioning(channel#34, id#35, sales#126, returns#127, profit#128, 5), ENSURE_REQUIREMENTS, [plan_id=44] +Arguments: hashpartitioning(channel#34, id#35, sales#126, returns#127, profit#128, 5), ENSURE_REQUIREMENTS, [plan_id=24] -(264) HashAggregate [codegen id : 64] +(116) HashAggregate [codegen id : 64] Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Keys [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Functions: [] Aggregate Attributes: [] Results [5]: [channel#34, id#35, sales#126, returns#127, profit#128] -(265) TakeOrderedAndProject +(117) TakeOrderedAndProject Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#126, returns#127, profit#128] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (270) -+- * ColumnarToRow (269) - +- CometProject (268) - +- CometFilter (267) - +- CometScan parquet spark_catalog.default.date_dim (266) +BroadcastExchange (122) ++- * ColumnarToRow (121) + +- CometProject (120) + +- CometFilter (119) + +- CometScan parquet spark_catalog.default.date_dim (118) (unknown) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#237] +Output [2]: [d_date_sk#14, d_date#171] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(267) CometFilter -Input [2]: [d_date_sk#14, d_date#237] -Condition : (((isnotnull(d_date#237) AND (d_date#237 >= 1998-08-04)) AND (d_date#237 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) +(119) CometFilter +Input [2]: [d_date_sk#14, d_date#171] +Condition : (((isnotnull(d_date#171) AND (d_date#171 >= 1998-08-04)) AND (d_date#171 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) -(268) CometProject -Input [2]: [d_date_sk#14, d_date#237] +(120) CometProject +Input [2]: [d_date_sk#14, d_date#171] Arguments: [d_date_sk#14], [d_date_sk#14] -(269) ColumnarToRow [codegen id : 1] +(121) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(270) BroadcastExchange +(122) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=45] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=25] Subquery:2 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 Subquery:3 Hosting operator id = 69 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#8 -Subquery:4 Hosting operator id = 103 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 - -Subquery:5 Hosting operator id = 127 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 - -Subquery:6 Hosting operator id = 151 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#8 - -Subquery:7 Hosting operator id = 182 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 - -Subquery:8 Hosting operator id = 206 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 - -Subquery:9 Hosting operator id = 230 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#8 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt index 332c26b97e..f6a37c6372 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt @@ -165,219 +165,13 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter - Exchange [channel,id] #19 - WholeStageCodegen (39) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (26) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #20 - WholeStageCodegen (25) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_ticket_number] - CometExchange [ss_item_sk,ss_ticket_number] #21 - CometFilter [ss_store_sk,ss_item_sk,ss_promo_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] #6 - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - ReusedExchange [s_store_sk,s_store_id] #7 - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (32) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #22 - WholeStageCodegen (31) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number] - CometExchange [cs_item_sk,cs_order_number] #23 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #12 - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (38) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #24 - WholeStageCodegen (37) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - CometSort [ws_item_sk,ws_order_number] - CometExchange [ws_item_sk,ws_order_number] #25 - CometFilter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] #16 - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - ReusedExchange [web_site_sk,web_site_id] #17 - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 WholeStageCodegen (62) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter - Exchange #26 + Exchange #19 WholeStageCodegen (61) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] InputAdapter - Exchange [channel,id] #27 - WholeStageCodegen (60) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (47) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #28 - WholeStageCodegen (46) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_ticket_number] - CometExchange [ss_item_sk,ss_ticket_number] #29 - CometFilter [ss_store_sk,ss_item_sk,ss_promo_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] #6 - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - ReusedExchange [s_store_sk,s_store_id] #7 - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (53) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #30 - WholeStageCodegen (52) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number] - CometExchange [cs_item_sk,cs_order_number] #31 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #12 - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (59) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #32 - WholeStageCodegen (58) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - CometSort [ws_item_sk,ws_order_number] - CometExchange [ws_item_sk,ws_order_number] #33 - CometFilter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] #16 - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - ReusedExchange [web_site_sk,web_site_id] #17 - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 From f17f9e35e3325718a3c1cf2dff8a26b84e338af4 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 25 Apr 2024 14:07:16 -0700 Subject: [PATCH 4/8] Restore plan stability --- .../shuffle/CometShuffleExchangeExec.scala | 3 ++ .../approved-plans-v1_4/q16/explain.txt | 14 +++---- .../approved-plans-v1_4/q23b/explain.txt | 20 +++++----- .../approved-plans-v1_4/q24a/explain.txt | 22 +++++------ .../approved-plans-v1_4/q24b/explain.txt | 22 +++++------ .../approved-plans-v1_4/q28/explain.txt | 34 ++++++++--------- .../approved-plans-v1_4/q40/explain.txt | 12 +++--- .../approved-plans-v1_4/q41/explain.txt | 6 +-- .../approved-plans-v1_4/q44/explain.txt | 8 ++-- .../approved-plans-v1_4/q54/explain.txt | 4 +- .../approved-plans-v1_4/q6/explain.txt | 12 +++--- .../approved-plans-v1_4/q64/explain.txt | 38 +++++++++---------- .../approved-plans-v1_4/q72/explain.txt | 6 +-- .../approved-plans-v1_4/q75/explain.txt | 32 ++++++++-------- .../approved-plans-v1_4/q78/explain.txt | 20 +++++----- .../approved-plans-v1_4/q80/explain.txt | 32 ++++++++-------- .../approved-plans-v1_4/q9/explain.txt | 10 ++--- .../approved-plans-v1_4/q93/explain.txt | 8 ++-- .../approved-plans-v1_4/q94/explain.txt | 14 +++---- .../approved-plans-v1_4/q95/explain.txt | 14 +++---- .../approved-plans-v2_7/q24/explain.txt | 24 ++++++------ .../approved-plans-v2_7/q6/explain.txt | 12 +++--- .../approved-plans-v2_7/q64/explain.txt | 38 +++++++++---------- .../approved-plans-v2_7/q72/explain.txt | 6 +-- .../approved-plans-v2_7/q75/explain.txt | 32 ++++++++-------- .../approved-plans-v2_7/q78/explain.txt | 20 +++++----- .../approved-plans-v2_7/q80a/explain.txt | 38 +++++++++---------- 27 files changed, 252 insertions(+), 249 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala index e05ac7a23a..fb2f2a2092 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala @@ -210,6 +210,9 @@ case class CometShuffleExchangeExec( override def hashCode(): Int = Objects.hashCode(outputPartitioning, shuffleOrigin, shuffleType, advisoryPartitionSize, child) + + override def stringArgs: Iterator[Any] = + Iterator(outputPartitioning, shuffleOrigin, shuffleType, child) ++ Iterator(s"[plan_id=$id]") } object CometShuffleExchangeExec extends ShimCometShuffleExchangeExec { diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt index 61cd5f78b4..1762c8c6c8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt @@ -63,7 +63,7 @@ Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_wareho (4) CometExchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), Exchange hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] @@ -84,7 +84,7 @@ Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order (9) CometExchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), Exchange hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [cs_warehouse_sk#9, cs_order_number#10] @@ -115,7 +115,7 @@ Arguments: [cr_order_number#12], [cr_order_number#12] (16) CometExchange Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), Exchange hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [cr_order_number#12] @@ -150,7 +150,7 @@ Input [1]: [d_date_sk#14] (24) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (25) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_ship_date_sk#1] @@ -182,7 +182,7 @@ Input [1]: [ca_address_sk#16] (31) BroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (32) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_ship_addr_sk#2] @@ -214,7 +214,7 @@ Input [1]: [cc_call_center_sk#18] (38) BroadcastExchange Input [1]: [cc_call_center_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (39) BroadcastHashJoin [codegen id : 8] Left keys [1]: [cs_call_center_sk#3] @@ -249,7 +249,7 @@ Results [3]: [sum#22, sum#23, count#25] (44) Exchange Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] (45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt index d60f0c4837..86227b8324 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt @@ -302,7 +302,7 @@ Condition : isnotnull(c_customer_sk#33) (45) CometExchange Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] -Arguments: hashpartitioning(c_customer_sk#33, 5), Exchange hashpartitioning(c_customer_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(c_customer_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (46) CometSort Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] @@ -341,7 +341,7 @@ Join condition: None (54) BroadcastExchange Input [3]: [c_customer_sk#33, c_first_name#34, c_last_name#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] (55) BroadcastHashJoin [codegen id : 16] Left keys [1]: [cs_bill_customer_sk#1] @@ -375,7 +375,7 @@ Results [4]: [c_last_name#35, c_first_name#34, sum#39, isEmpty#40] (61) Exchange Input [4]: [c_last_name#35, c_first_name#34, sum#39, isEmpty#40] -Arguments: hashpartitioning(c_last_name#35, c_first_name#34, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(c_last_name#35, c_first_name#34, 5), ENSURE_REQUIREMENTS, [plan_id=9] (62) HashAggregate [codegen id : 17] Input [4]: [c_last_name#35, c_first_name#34, sum#39, isEmpty#40] @@ -414,7 +414,7 @@ Input [5]: [ws_item_sk#43, ws_bill_customer_sk#44, ws_quantity#45, ws_list_price (69) Exchange Input [4]: [ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47] -Arguments: hashpartitioning(ws_bill_customer_sk#44, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(ws_bill_customer_sk#44, 5), ENSURE_REQUIREMENTS, [plan_id=10] (70) Sort [codegen id : 23] Input [4]: [ws_bill_customer_sk#44, ws_quantity#45, ws_list_price#46, ws_sold_date_sk#47] @@ -483,7 +483,7 @@ Results [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] (84) Exchange Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=11] (85) HashAggregate [codegen id : 34] Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] @@ -528,7 +528,7 @@ Input [1]: [d_date_sk#36] (92) BroadcastExchange Input [1]: [d_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] Subquery:2 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 BroadcastExchange (97) @@ -558,7 +558,7 @@ Input [2]: [d_date_sk#10, d_date#11] (97) BroadcastExchange Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] Subquery:3 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquery#31, [id=#32] * HashAggregate (112) @@ -628,7 +628,7 @@ Results [3]: [c_customer_sk#67, sum#71, isEmpty#72] (108) Exchange Input [3]: [c_customer_sk#67, sum#71, isEmpty#72] -Arguments: hashpartitioning(c_customer_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(c_customer_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=14] (109) HashAggregate [codegen id : 4] Input [3]: [c_customer_sk#67, sum#71, isEmpty#72] @@ -646,7 +646,7 @@ Results [1]: [max#76] (111) Exchange Input [1]: [max#76] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] (112) HashAggregate [codegen id : 5] Input [1]: [max#76] @@ -683,7 +683,7 @@ Input [1]: [d_date_sk#68] (117) BroadcastExchange Input [1]: [d_date_sk#68] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=16] Subquery:5 Hosting operator id = 50 Hosting Expression = ReusedSubquery Subquery scalar-subquery#31, [id=#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt index 9e2b9f9f65..6e03148d53 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt @@ -63,7 +63,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s (4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -86,7 +86,7 @@ Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8 (9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), Exchange hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -124,7 +124,7 @@ Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (18) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#3] @@ -152,7 +152,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (24) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] (25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -180,7 +180,7 @@ Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#2 (30) BroadcastExchange Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (31) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_customer_sk#2] @@ -208,7 +208,7 @@ Input [3]: [ca_state#25, ca_zip#26, ca_country#27] (36) BroadcastExchange Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] (37) BroadcastHashJoin [codegen id : 5] Left keys [2]: [c_birth_country#24, s_zip#14] @@ -229,7 +229,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (40) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] (41) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] @@ -247,7 +247,7 @@ Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty# (43) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] (44) HashAggregate [codegen id : 7] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] @@ -348,7 +348,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (59) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (60) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -395,7 +395,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (69) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] (70) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] @@ -413,7 +413,7 @@ Results [2]: [sum#44, count#45] (72) Exchange Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] (73) HashAggregate [codegen id : 7] Input [2]: [sum#44, count#45] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt index f7aa1c2bc8..fb3113df00 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt @@ -63,7 +63,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s (4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -86,7 +86,7 @@ Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8 (9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), Exchange hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -124,7 +124,7 @@ Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (18) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#3] @@ -152,7 +152,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (24) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] (25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -180,7 +180,7 @@ Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#2 (30) BroadcastExchange Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (31) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_customer_sk#2] @@ -208,7 +208,7 @@ Input [3]: [ca_state#25, ca_zip#26, ca_country#27] (36) BroadcastExchange Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] (37) BroadcastHashJoin [codegen id : 5] Left keys [2]: [c_birth_country#24, s_zip#14] @@ -229,7 +229,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (40) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] (41) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] @@ -247,7 +247,7 @@ Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty# (43) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] (44) HashAggregate [codegen id : 7] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] @@ -348,7 +348,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (59) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (60) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -395,7 +395,7 @@ Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_ (69) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] (70) HashAggregate [codegen id : 6] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] @@ -413,7 +413,7 @@ Results [2]: [sum#44, count#45] (72) Exchange Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] (73) HashAggregate [codegen id : 7] Input [2]: [sum#44, count#45] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt index 2b67b52a18..db6d112e64 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt @@ -93,7 +93,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_li (5) CometExchange Input [4]: [ss_list_price#3, sum#6, count#7, count#8] -Arguments: hashpartitioning(ss_list_price#3, 5), Exchange hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (6) ColumnarToRow [codegen id : 1] Input [4]: [ss_list_price#3, sum#6, count#7, count#8] @@ -114,7 +114,7 @@ Results [4]: [sum#6, count#7, count#8, count#12] (9) Exchange Input [4]: [sum#6, count#7, count#8, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] (10) HashAggregate [codegen id : 12] Input [4]: [sum#6, count#7, count#8, count#12] @@ -145,7 +145,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_l (15) CometExchange Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -Arguments: hashpartitioning(ss_list_price#18, 5), Exchange hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=4], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (16) ColumnarToRow [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] @@ -166,7 +166,7 @@ Results [4]: [sum#21, count#22, count#23, count#27] (19) Exchange Input [4]: [sum#21, count#22, count#23, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] (20) HashAggregate [codegen id : 3] Input [4]: [sum#21, count#22, count#23, count#27] @@ -177,7 +177,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal( (21) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] -Arguments: IdentityBroadcastMode, [plan_id=7] +Arguments: IdentityBroadcastMode, [plan_id=5] (22) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -205,7 +205,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_l (27) CometExchange Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -Arguments: hashpartitioning(ss_list_price#33, 5), Exchange hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=8], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (28) ColumnarToRow [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] @@ -226,7 +226,7 @@ Results [4]: [sum#36, count#37, count#38, count#42] (31) Exchange Input [4]: [sum#36, count#37, count#38, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] (32) HashAggregate [codegen id : 5] Input [4]: [sum#36, count#37, count#38, count#42] @@ -237,7 +237,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal( (33) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] -Arguments: IdentityBroadcastMode, [plan_id=11] +Arguments: IdentityBroadcastMode, [plan_id=8] (34) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -265,7 +265,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_l (39) CometExchange Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -Arguments: hashpartitioning(ss_list_price#48, 5), Exchange hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=12], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (40) ColumnarToRow [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] @@ -286,7 +286,7 @@ Results [4]: [sum#51, count#52, count#53, count#57] (43) Exchange Input [4]: [sum#51, count#52, count#53, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] (44) HashAggregate [codegen id : 7] Input [4]: [sum#51, count#52, count#53, count#57] @@ -297,7 +297,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal( (45) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] -Arguments: IdentityBroadcastMode, [plan_id=15] +Arguments: IdentityBroadcastMode, [plan_id=11] (46) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -325,7 +325,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_l (51) CometExchange Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -Arguments: hashpartitioning(ss_list_price#63, 5), Exchange hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=16], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=17] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (52) ColumnarToRow [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] @@ -346,7 +346,7 @@ Results [4]: [sum#66, count#67, count#68, count#72] (55) Exchange Input [4]: [sum#66, count#67, count#68, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] (56) HashAggregate [codegen id : 9] Input [4]: [sum#66, count#67, count#68, count#72] @@ -357,7 +357,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal( (57) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] -Arguments: IdentityBroadcastMode, [plan_id=19] +Arguments: IdentityBroadcastMode, [plan_id=14] (58) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner @@ -385,7 +385,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_l (63) CometExchange Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -Arguments: hashpartitioning(ss_list_price#78, 5), Exchange hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=20], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=21] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (64) ColumnarToRow [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] @@ -406,7 +406,7 @@ Results [4]: [sum#81, count#82, count#83, count#87] (67) Exchange Input [4]: [sum#81, count#82, count#83, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=22] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] (68) HashAggregate [codegen id : 11] Input [4]: [sum#81, count#82, count#83, count#87] @@ -417,7 +417,7 @@ Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal( (69) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] -Arguments: IdentityBroadcastMode, [plan_id=23] +Arguments: IdentityBroadcastMode, [plan_id=17] (70) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt index e0c36ee89c..9f6036a23d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt @@ -47,7 +47,7 @@ Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) (3) CometExchange Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), Exchange hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (4) CometSort Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] @@ -70,7 +70,7 @@ Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_item_sk#7, (8) CometExchange Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), Exchange hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] @@ -104,7 +104,7 @@ Input [2]: [w_warehouse_sk#11, w_state#12] (16) BroadcastExchange Input [2]: [w_warehouse_sk#11, w_state#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] (17) BroadcastHashJoin [codegen id : 4] Left keys [1]: [cs_warehouse_sk#1] @@ -136,7 +136,7 @@ Input [2]: [i_item_sk#13, i_item_id#14] (23) BroadcastExchange Input [2]: [i_item_sk#13, i_item_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (24) BroadcastHashJoin [codegen id : 4] Left keys [1]: [cs_item_sk#2] @@ -170,7 +170,7 @@ Results [6]: [w_state#12, i_item_id#14, sum#22, isEmpty#23, sum#24, isEmpty#25] (30) Exchange Input [6]: [w_state#12, i_item_id#14, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, [plan_id=5] (31) HashAggregate [codegen id : 5] Input [6]: [w_state#12, i_item_id#14, sum#22, isEmpty#23, sum#24, isEmpty#25] @@ -208,6 +208,6 @@ Input [2]: [d_date_sk#16, d_date#17] (36) BroadcastExchange Input [2]: [d_date_sk#16, d_date#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt index 69f22dcdc1..2308990771 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt @@ -61,7 +61,7 @@ Functions [1]: [partial_count(1)] (9) CometExchange Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), Exchange hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [2]: [i_manufact#5, count#9] @@ -81,7 +81,7 @@ Input [1]: [i_manufact#5] (14) BroadcastExchange Input [1]: [i_manufact#5] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] (15) BroadcastHashJoin [codegen id : 2] Left keys [1]: [i_manufact#2] @@ -102,7 +102,7 @@ Results [1]: [i_product_name#3] (18) Exchange Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [plan_id=3] (19) HashAggregate [codegen id : 3] Input [1]: [i_product_name#3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt index f0e9cc19d5..d7f13b274c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt @@ -56,7 +56,7 @@ Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] (5) CometExchange Input [3]: [ss_item_sk#1, sum#5, count#6] -Arguments: hashpartitioning(ss_item_sk#1, 5), Exchange hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (6) CometHashAggregate Input [3]: [ss_item_sk#1, sum#5, count#6] @@ -69,7 +69,7 @@ Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 (8) CometExchange Input [2]: [item_sk#7, rank_col#8] -Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [2]: [item_sk#7, rank_col#8] @@ -146,7 +146,7 @@ Input [2]: [i_item_sk#15, i_product_name#16] (27) BroadcastExchange Input [2]: [i_item_sk#15, i_product_name#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] (28) BroadcastHashJoin [codegen id : 7] Left keys [1]: [item_sk#7] @@ -209,7 +209,7 @@ Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#23))] (38) CometExchange Input [3]: [ss_store_sk#22, sum#25, count#26] -Arguments: hashpartitioning(ss_store_sk#22, 5), Exchange hashpartitioning(ss_store_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(ss_store_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (39) CometHashAggregate Input [3]: [ss_store_sk#22, sum#25, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt index 174a142f14..00c8f9c9b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt @@ -420,7 +420,7 @@ Functions: [] (70) CometExchange Input [1]: [(d_month_seq + 1)#49] -Arguments: hashpartitioning((d_month_seq + 1)#49, 5), Exchange hashpartitioning((d_month_seq + 1)#49, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning((d_month_seq + 1)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (71) CometHashAggregate Input [1]: [(d_month_seq + 1)#49] @@ -462,7 +462,7 @@ Functions: [] (77) CometExchange Input [1]: [(d_month_seq + 3)#53] -Arguments: hashpartitioning((d_month_seq + 3)#53, 5), Exchange hashpartitioning((d_month_seq + 3)#53, 5), ENSURE_REQUIREMENTS, [plan_id=13], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Arguments: hashpartitioning((d_month_seq + 3)#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (78) CometHashAggregate Input [1]: [(d_month_seq + 3)#53] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt index 0a63dcb589..5f34aa3df9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt @@ -156,7 +156,7 @@ Functions [1]: [partial_avg(UnscaledValue(i_current_price#13))] (25) CometExchange Input [3]: [i_category#14, sum#15, count#16] -Arguments: hashpartitioning(i_category#14, 5), Exchange hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate Input [3]: [i_category#14, sum#15, count#16] @@ -172,7 +172,7 @@ Input [2]: [avg(i_current_price)#17, i_category#14] (29) BroadcastExchange Input [2]: [avg(i_current_price)#17, i_category#14] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] (30) BroadcastHashJoin [codegen id : 5] Left keys [1]: [i_category#12] @@ -186,7 +186,7 @@ Input [5]: [i_item_sk#10, i_current_price#11, i_category#12, avg(i_current_price (32) BroadcastExchange Input [1]: [i_item_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (33) BroadcastHashJoin [codegen id : 6] Left keys [1]: [ss_item_sk#5] @@ -207,7 +207,7 @@ Results [2]: [ca_state#2, count#19] (36) Exchange Input [2]: [ca_state#2, count#19] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=6] (37) HashAggregate [codegen id : 7] Input [2]: [ca_state#2, count#19] @@ -254,7 +254,7 @@ Input [1]: [d_date_sk#9] (44) BroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] Subquery:2 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#24, [id=#25] * ColumnarToRow (51) @@ -288,7 +288,7 @@ Functions: [] (49) CometExchange Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), Exchange hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (50) CometHashAggregate Input [1]: [d_month_seq#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt index c7a7f93d5a..c21eda6be3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt @@ -222,7 +222,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), Exchange hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -245,7 +245,7 @@ Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_s (14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), Exchange hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -268,7 +268,7 @@ Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_ (19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), Exchange hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -290,7 +290,7 @@ Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum((( (24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), Exchange hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -350,7 +350,7 @@ Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] (38) BroadcastExchange Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (39) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_store_sk#6] @@ -378,7 +378,7 @@ Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_cu (44) BroadcastExchange Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] (45) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_customer_sk#2] @@ -406,7 +406,7 @@ Input [2]: [d_date_sk#43, d_year#44] (50) BroadcastExchange Input [2]: [d_date_sk#43, d_year#44] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] (51) BroadcastHashJoin [codegen id : 16] Left keys [1]: [c_first_sales_date_sk#42] @@ -447,7 +447,7 @@ Input [2]: [cd_demo_sk#47, cd_marital_status#48] (59) BroadcastExchange Input [2]: [cd_demo_sk#47, cd_marital_status#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] (60) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_cdemo_sk#3] @@ -488,7 +488,7 @@ Input [1]: [p_promo_sk#51] (68) BroadcastExchange Input [1]: [p_promo_sk#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (69) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_promo_sk#7] @@ -516,7 +516,7 @@ Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] (74) BroadcastExchange Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] (75) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_hdemo_sk#4] @@ -557,7 +557,7 @@ Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59 (83) BroadcastExchange Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] (84) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_addr_sk#5] @@ -598,7 +598,7 @@ Input [1]: [ib_income_band_sk#66] (92) BroadcastExchange Input [1]: [ib_income_band_sk#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] (93) BroadcastHashJoin [codegen id : 16] Left keys [1]: [hd_income_band_sk#53] @@ -643,7 +643,7 @@ Input [2]: [i_item_sk#68, i_product_name#71] (102) BroadcastExchange Input [2]: [i_item_sk#68, i_product_name#71] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] (103) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_item_sk#1] @@ -671,7 +671,7 @@ Results [17]: [i_product_name#71 AS product_name#84, i_item_sk#68 AS item_sk#85, (107) Exchange Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] -Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=14] (108) Sort [codegen id : 17] Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] @@ -719,7 +719,7 @@ Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104 (117) CometExchange Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -Arguments: hashpartitioning(ss_item_sk#101, 5), Exchange hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=19], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=20] +Arguments: hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (118) CometSort Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] @@ -968,7 +968,7 @@ Results [8]: [i_item_sk#161 AS item_sk#169, s_store_name#128 AS store_name#170, (174) Exchange Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=21] +Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=16] (175) Sort [codegen id : 34] Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] @@ -986,7 +986,7 @@ Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_ (178) Exchange Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=22] +Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=17] (179) Sort [codegen id : 36] Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] @@ -1017,7 +1017,7 @@ Input [2]: [d_date_sk#32, d_year#33] (183) BroadcastExchange Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=23] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] Subquery:2 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#112 IN dynamicpruning#113 BroadcastExchange (187) @@ -1042,6 +1042,6 @@ Input [2]: [d_date_sk#125, d_year#126] (187) BroadcastExchange Input [2]: [d_date_sk#125, d_year#126] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=24] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt index 30e51d82d4..c0d1f949a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt @@ -357,7 +357,7 @@ Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number# (62) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), Exchange hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=10], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (63) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] @@ -385,7 +385,7 @@ Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] (68) Exchange Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=11] (69) HashAggregate [codegen id : 14] Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] @@ -428,6 +428,6 @@ Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] (75) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index 5c7ccc79eb..3a29db88ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -218,7 +218,7 @@ Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_ (19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), Exchange hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -280,7 +280,7 @@ Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_pr (33) Exchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] (34) Sort [codegen id : 10] Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -303,7 +303,7 @@ Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return (38) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), Exchange hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (39) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -365,7 +365,7 @@ Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_pri (52) Exchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=6] (53) Sort [codegen id : 16] Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -388,7 +388,7 @@ Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_ (57) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), Exchange hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (58) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -418,7 +418,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (64) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=8] (65) HashAggregate [codegen id : 20] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -436,7 +436,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (67) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=9] (68) HashAggregate [codegen id : 21] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] @@ -451,7 +451,7 @@ Condition : isnotnull(sales_cnt#68) (70) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=10] (71) Sort [codegen id : 22] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] @@ -500,7 +500,7 @@ Input [11]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_pri (81) Exchange Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=11] (82) Sort [codegen id : 26] Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] @@ -569,7 +569,7 @@ Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_pr (97) Exchange Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=12] (98) Sort [codegen id : 32] Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] @@ -638,7 +638,7 @@ Input [11]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_ (113) Exchange Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=16] +Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=13] (114) Sort [codegen id : 38] Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] @@ -675,7 +675,7 @@ Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manuf (122) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=17] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=14] (123) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] @@ -693,7 +693,7 @@ Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manuf (125) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=15] (126) HashAggregate [codegen id : 43] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] @@ -708,7 +708,7 @@ Condition : isnotnull(sales_cnt#123) (128) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=19] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=16] (129) Sort [codegen id : 44] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] @@ -753,7 +753,7 @@ Input [2]: [d_date_sk#13, d_year#14] (136) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 @@ -782,7 +782,7 @@ Input [2]: [d_date_sk#81, d_year#82] (140) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=21] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt index 25ac63aca3..caab92168f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt @@ -82,7 +82,7 @@ Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) (3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -105,7 +105,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number# (8) CometExchange Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), Exchange hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [2]: [sr_item_sk#9, sr_ticket_number#10] @@ -149,7 +149,7 @@ Results [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] (18) Exchange Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] (19) HashAggregate [codegen id : 3] Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] @@ -176,7 +176,7 @@ Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) (23) CometExchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), Exchange hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (24) CometSort Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] @@ -199,7 +199,7 @@ Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number# (28) CometExchange Input [2]: [wr_item_sk#35, wr_order_number#36] -Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), Exchange hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=8], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (29) CometSort Input [2]: [wr_item_sk#35, wr_order_number#36] @@ -243,7 +243,7 @@ Results [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, (38) Exchange Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=6] (39) HashAggregate [codegen id : 6] Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] @@ -284,7 +284,7 @@ Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) (46) CometExchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), Exchange hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (47) CometSort Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] @@ -307,7 +307,7 @@ Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number# (51) CometExchange Input [2]: [cr_item_sk#62, cr_order_number#63] -Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), Exchange hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, [plan_id=13], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (52) CometSort Input [2]: [cr_item_sk#62, cr_order_number#63] @@ -351,7 +351,7 @@ Results [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, (61) Exchange Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=9] (62) HashAggregate [codegen id : 10] Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] @@ -407,7 +407,7 @@ Input [2]: [d_date_sk#12, d_year#13] (71) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] Subquery:2 Hosting operator id = 21 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt index 39b431840c..dfd123a1d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt @@ -119,7 +119,7 @@ Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnul (3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -142,7 +142,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] (8) CometExchange Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), Exchange hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] @@ -189,7 +189,7 @@ Input [2]: [s_store_sk#15, s_store_id#16] (19) BroadcastExchange Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] (20) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#2] @@ -221,7 +221,7 @@ Input [1]: [i_item_sk#17] (26) BroadcastExchange Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (27) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -253,7 +253,7 @@ Input [1]: [p_promo_sk#19] (33) BroadcastExchange Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_promo_sk#3] @@ -274,7 +274,7 @@ Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] (37) Exchange Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=6] (38) HashAggregate [codegen id : 6] Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] @@ -297,7 +297,7 @@ Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND (41) CometExchange Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (42) CometSort Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] @@ -320,7 +320,7 @@ Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss# (46) CometExchange Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), Exchange hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (47) CometSort Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] @@ -367,7 +367,7 @@ Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] (57) BroadcastExchange Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (58) BroadcastHashJoin [codegen id : 11] Left keys [1]: [cs_catalog_page_sk#39] @@ -414,7 +414,7 @@ Results [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty (67) Exchange Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] -Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=10] (68) HashAggregate [codegen id : 12] Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] @@ -437,7 +437,7 @@ Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isn (71) CometExchange Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] +Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (72) CometSort Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] @@ -460,7 +460,7 @@ Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] (76) CometExchange Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), Exchange hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, [plan_id=17], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] +Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (77) CometSort Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] @@ -507,7 +507,7 @@ Input [2]: [web_site_sk#89, web_site_id#90] (87) BroadcastExchange Input [2]: [web_site_sk#89, web_site_id#90] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] (88) BroadcastHashJoin [codegen id : 17] Left keys [1]: [ws_web_site_sk#76] @@ -554,7 +554,7 @@ Results [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (97) Exchange Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=20] +Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=14] (98) HashAggregate [codegen id : 18] Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] @@ -578,7 +578,7 @@ Results [9]: [channel#111, id#112, spark_grouping_id#113, sum#120, isEmpty#121, (102) Exchange Input [9]: [channel#111, id#112, spark_grouping_id#113, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] -Arguments: hashpartitioning(channel#111, id#112, spark_grouping_id#113, 5), ENSURE_REQUIREMENTS, [plan_id=21] +Arguments: hashpartitioning(channel#111, id#112, spark_grouping_id#113, 5), ENSURE_REQUIREMENTS, [plan_id=15] (103) HashAggregate [codegen id : 20] Input [9]: [channel#111, id#112, spark_grouping_id#113, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] @@ -621,7 +621,7 @@ Input [1]: [d_date_sk#14] (109) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=22] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=16] Subquery:2 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt index d66aa0fbef..1671811424 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt @@ -58,7 +58,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (9) CometExchange Input [5]: [count#21, sum#22, count#23, sum#24, count#25] -Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [5]: [count#21, sum#22, count#23, sum#24, count#25] @@ -109,7 +109,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (17) CometExchange Input [5]: [count#34, sum#35, count#36, sum#37, count#38] -Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate Input [5]: [count#34, sum#35, count#36, sum#37, count#38] @@ -160,7 +160,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (25) CometExchange Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate Input [5]: [count#47, sum#48, count#49, sum#50, count#51] @@ -211,7 +211,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (33) CometExchange Input [5]: [count#60, sum#61, count#62, sum#63, count#64] -Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (34) CometHashAggregate Input [5]: [count#60, sum#61, count#62, sum#63, count#64] @@ -262,7 +262,7 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# (41) CometExchange Input [5]: [count#73, sum#74, count#75, sum#76, count#77] -Arguments: SinglePartition, Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (42) CometHashAggregate Input [5]: [count#73, sum#74, count#75, sum#76, count#77] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt index 5e011dac84..4b396dbc38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt @@ -36,7 +36,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s (3) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (4) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] @@ -59,7 +59,7 @@ Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity (8) CometExchange Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), Exchange hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] @@ -97,7 +97,7 @@ Input [1]: [r_reason_sk#12] (17) BroadcastExchange Input [1]: [r_reason_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (18) BroadcastHashJoin [codegen id : 2] Left keys [1]: [sr_reason_sk#8] @@ -118,7 +118,7 @@ Results [3]: [ss_customer_sk#2, sum#17, isEmpty#18] (21) Exchange Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] (22) HashAggregate [codegen id : 3] Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt index 8c52d67797..aafe59a77e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt @@ -63,7 +63,7 @@ Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse (4) CometExchange Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: hashpartitioning(ws_order_number#5, 5), Exchange hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] @@ -84,7 +84,7 @@ Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order (9) CometExchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), Exchange hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [ws_warehouse_sk#9, ws_order_number#10] @@ -115,7 +115,7 @@ Arguments: [wr_order_number#12], [wr_order_number#12] (16) CometExchange Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), Exchange hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [wr_order_number#12] @@ -150,7 +150,7 @@ Input [1]: [d_date_sk#14] (24) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (25) BroadcastHashJoin [codegen id : 8] Left keys [1]: [ws_ship_date_sk#1] @@ -182,7 +182,7 @@ Input [1]: [ca_address_sk#16] (31) BroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (32) BroadcastHashJoin [codegen id : 8] Left keys [1]: [ws_ship_addr_sk#2] @@ -214,7 +214,7 @@ Input [1]: [web_site_sk#18] (38) BroadcastExchange Input [1]: [web_site_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (39) BroadcastHashJoin [codegen id : 8] Left keys [1]: [ws_web_site_sk#3] @@ -249,7 +249,7 @@ Results [3]: [sum#22, sum#23, count#25] (44) Exchange Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] (45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt index 06488a31db..375c7bf2f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt @@ -79,7 +79,7 @@ Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_num (4) CometExchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), Exchange hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] @@ -105,7 +105,7 @@ Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_ (10) CometExchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), Exchange hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (11) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] @@ -157,7 +157,7 @@ Arguments: [wr_order_number#13], [wr_order_number#13] (22) CometExchange Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), Exchange hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometSort Input [1]: [wr_order_number#13] @@ -232,7 +232,7 @@ Input [1]: [d_date_sk#15] (40) BroadcastExchange Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_ship_date_sk#1] @@ -264,7 +264,7 @@ Input [1]: [ca_address_sk#17] (47) BroadcastExchange Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (48) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_ship_addr_sk#2] @@ -296,7 +296,7 @@ Input [1]: [web_site_sk#19] (54) BroadcastExchange Input [1]: [web_site_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (55) BroadcastHashJoin [codegen id : 14] Left keys [1]: [ws_web_site_sk#3] @@ -331,7 +331,7 @@ Results [3]: [sum#23, sum#24, count#26] (60) Exchange Input [3]: [sum#23, sum#24, count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] (61) HashAggregate [codegen id : 15] Input [3]: [sum#23, sum#24, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt index 6101b5e09e..9bbf5e9847 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt @@ -65,7 +65,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s (4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -88,7 +88,7 @@ Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8 (9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), Exchange hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -126,7 +126,7 @@ Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] (18) BroadcastExchange Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#3] @@ -154,7 +154,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (24) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] (25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -182,7 +182,7 @@ Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name (30) BroadcastExchange Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (31) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_customer_sk#2] @@ -210,7 +210,7 @@ Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] (36) BroadcastExchange Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=6] (37) BroadcastHashJoin [codegen id : 5] Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] @@ -231,7 +231,7 @@ Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_ (40) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] (41) HashAggregate [codegen id : 6] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] @@ -249,7 +249,7 @@ Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty# (43) Exchange Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] (44) HashAggregate [codegen id : 7] Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] @@ -264,7 +264,7 @@ Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subqu (46) Exchange Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=9] (47) Sort [codegen id : 8] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] @@ -358,7 +358,7 @@ Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, (61) BroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] (62) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -405,7 +405,7 @@ Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_ (71) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=11] (72) HashAggregate [codegen id : 6] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] @@ -423,7 +423,7 @@ Results [2]: [sum#46, count#47] (74) Exchange Input [2]: [sum#46, count#47] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] (75) HashAggregate [codegen id : 7] Input [2]: [sum#46, count#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt index 78abb49dee..fcaa84890b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt @@ -156,7 +156,7 @@ Functions [1]: [partial_avg(UnscaledValue(i_current_price#13))] (25) CometExchange Input [3]: [i_category#14, sum#15, count#16] -Arguments: hashpartitioning(i_category#14, 5), Exchange hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate Input [3]: [i_category#14, sum#15, count#16] @@ -172,7 +172,7 @@ Input [2]: [avg(i_current_price)#17, i_category#14] (29) BroadcastExchange Input [2]: [avg(i_current_price)#17, i_category#14] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] (30) BroadcastHashJoin [codegen id : 5] Left keys [1]: [i_category#12] @@ -186,7 +186,7 @@ Input [5]: [i_item_sk#10, i_current_price#11, i_category#12, avg(i_current_price (32) BroadcastExchange Input [1]: [i_item_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (33) BroadcastHashJoin [codegen id : 6] Left keys [1]: [ss_item_sk#5] @@ -207,7 +207,7 @@ Results [2]: [ca_state#2, count#19] (36) Exchange Input [2]: [ca_state#2, count#19] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=6] (37) HashAggregate [codegen id : 7] Input [2]: [ca_state#2, count#19] @@ -254,7 +254,7 @@ Input [1]: [d_date_sk#9] (44) BroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] Subquery:2 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#24, [id=#25] * ColumnarToRow (51) @@ -288,7 +288,7 @@ Functions: [] (49) CometExchange Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), Exchange hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (50) CometHashAggregate Input [1]: [d_month_seq#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt index a33f704315..1429e39daa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt @@ -222,7 +222,7 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add (9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), Exchange hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -245,7 +245,7 @@ Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_s (14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), Exchange hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -268,7 +268,7 @@ Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_ (19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), Exchange hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, [plan_id=5], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -290,7 +290,7 @@ Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum((( (24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), Exchange hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=7], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -350,7 +350,7 @@ Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] (38) BroadcastExchange Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] (39) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_store_sk#6] @@ -378,7 +378,7 @@ Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_cu (44) BroadcastExchange Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] (45) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_customer_sk#2] @@ -406,7 +406,7 @@ Input [2]: [d_date_sk#43, d_year#44] (50) BroadcastExchange Input [2]: [d_date_sk#43, d_year#44] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] (51) BroadcastHashJoin [codegen id : 16] Left keys [1]: [c_first_sales_date_sk#42] @@ -447,7 +447,7 @@ Input [2]: [cd_demo_sk#47, cd_marital_status#48] (59) BroadcastExchange Input [2]: [cd_demo_sk#47, cd_marital_status#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] (60) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_cdemo_sk#3] @@ -488,7 +488,7 @@ Input [1]: [p_promo_sk#51] (68) BroadcastExchange Input [1]: [p_promo_sk#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (69) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_promo_sk#7] @@ -516,7 +516,7 @@ Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] (74) BroadcastExchange Input [2]: [hd_demo_sk#52, hd_income_band_sk#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] (75) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_hdemo_sk#4] @@ -557,7 +557,7 @@ Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59 (83) BroadcastExchange Input [5]: [ca_address_sk#56, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] (84) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_addr_sk#5] @@ -598,7 +598,7 @@ Input [1]: [ib_income_band_sk#66] (92) BroadcastExchange Input [1]: [ib_income_band_sk#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] (93) BroadcastHashJoin [codegen id : 16] Left keys [1]: [hd_income_band_sk#53] @@ -643,7 +643,7 @@ Input [2]: [i_item_sk#68, i_product_name#71] (102) BroadcastExchange Input [2]: [i_item_sk#68, i_product_name#71] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] (103) BroadcastHashJoin [codegen id : 16] Left keys [1]: [ss_item_sk#1] @@ -671,7 +671,7 @@ Results [17]: [i_product_name#71 AS product_name#84, i_item_sk#68 AS item_sk#85, (107) Exchange Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] -Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(item_sk#85, store_name#86, store_zip#87, 5), ENSURE_REQUIREMENTS, [plan_id=14] (108) Sort [codegen id : 17] Input [17]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100] @@ -719,7 +719,7 @@ Arguments: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104 (117) CometExchange Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] -Arguments: hashpartitioning(ss_item_sk#101, 5), Exchange hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, [plan_id=19], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=20] +Arguments: hashpartitioning(ss_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (118) CometSort Input [11]: [ss_item_sk#101, ss_customer_sk#102, ss_cdemo_sk#103, ss_hdemo_sk#104, ss_addr_sk#105, ss_store_sk#106, ss_promo_sk#107, ss_wholesale_cost#109, ss_list_price#110, ss_coupon_amt#111, ss_sold_date_sk#112] @@ -968,7 +968,7 @@ Results [8]: [i_item_sk#161 AS item_sk#169, s_store_name#128 AS store_name#170, (174) Exchange Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] -Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=21] +Arguments: hashpartitioning(item_sk#169, store_name#170, store_zip#171, 5), ENSURE_REQUIREMENTS, [plan_id=16] (175) Sort [codegen id : 34] Input [8]: [item_sk#169, store_name#170, store_zip#171, syear#172, cnt#173, s1#174, s2#175, s3#176] @@ -986,7 +986,7 @@ Input [25]: [product_name#84, item_sk#85, store_name#86, store_zip#87, b_street_ (178) Exchange Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] -Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#174 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=22] +Arguments: rangepartitioning(product_name#84 ASC NULLS FIRST, store_name#86 ASC NULLS FIRST, cnt#173 ASC NULLS FIRST, s1#98 ASC NULLS FIRST, s1#174 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=17] (179) Sort [codegen id : 36] Input [21]: [product_name#84, store_name#86, store_zip#87, b_street_number#88, b_streen_name#89, b_city#90, b_zip#91, c_street_number#92, c_street_name#93, c_city#94, c_zip#95, syear#96, cnt#97, s1#98, s2#99, s3#100, s1#174, s2#175, s3#176, syear#172, cnt#173] @@ -1017,7 +1017,7 @@ Input [2]: [d_date_sk#32, d_year#33] (183) BroadcastExchange Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=23] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] Subquery:2 Hosting operator id = 109 Hosting Expression = ss_sold_date_sk#112 IN dynamicpruning#113 BroadcastExchange (187) @@ -1042,6 +1042,6 @@ Input [2]: [d_date_sk#125, d_year#126] (187) BroadcastExchange Input [2]: [d_date_sk#125, d_year#126] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=24] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt index 0da99bb7dc..46b9e51d5e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt @@ -357,7 +357,7 @@ Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number# (62) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), Exchange hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=10], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (63) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] @@ -385,7 +385,7 @@ Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] (68) Exchange Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=11] (69) HashAggregate [codegen id : 14] Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] @@ -428,6 +428,6 @@ Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] (75) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index c76c310164..3922f7efe6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -218,7 +218,7 @@ Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_ (19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), Exchange hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -280,7 +280,7 @@ Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_pr (33) Exchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] (34) Sort [codegen id : 10] Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -303,7 +303,7 @@ Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return (38) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), Exchange hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (39) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -365,7 +365,7 @@ Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_pri (52) Exchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=6] (53) Sort [codegen id : 16] Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -388,7 +388,7 @@ Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_ (57) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), Exchange hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (58) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -418,7 +418,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (64) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=8] (65) HashAggregate [codegen id : 20] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -436,7 +436,7 @@ Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufac (67) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=12] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=9] (68) HashAggregate [codegen id : 21] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] @@ -451,7 +451,7 @@ Condition : isnotnull(sales_cnt#68) (70) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=10] (71) Sort [codegen id : 22] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] @@ -500,7 +500,7 @@ Input [11]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_pri (81) Exchange Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, [plan_id=11] (82) Sort [codegen id : 26] Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] @@ -569,7 +569,7 @@ Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_pr (97) Exchange Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=12] (98) Sort [codegen id : 32] Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] @@ -638,7 +638,7 @@ Input [11]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_ (113) Exchange Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] -Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=16] +Arguments: hashpartitioning(ws_order_number#105, ws_item_sk#104, 5), ENSURE_REQUIREMENTS, [plan_id=13] (114) Sort [codegen id : 38] Input [9]: [ws_item_sk#104, ws_order_number#105, ws_quantity#106, ws_ext_sales_price#107, i_brand_id#111, i_class_id#112, i_category_id#113, i_manufact_id#114, d_year#116] @@ -675,7 +675,7 @@ Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manuf (122) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=17] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, [plan_id=14] (123) HashAggregate [codegen id : 42] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#20, sales_amt#21] @@ -693,7 +693,7 @@ Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manuf (125) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=18] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=15] (126) HashAggregate [codegen id : 43] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#64, sum#122] @@ -708,7 +708,7 @@ Condition : isnotnull(sales_cnt#123) (128) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=19] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=16] (129) Sort [codegen id : 44] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#123, sales_amt#124] @@ -753,7 +753,7 @@ Input [2]: [d_date_sk#13, d_year#14] (136) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 @@ -782,7 +782,7 @@ Input [2]: [d_date_sk#81, d_year#82] (140) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=21] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt index 3d7e571162..cfb9ed7d0e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt @@ -82,7 +82,7 @@ Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) (3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), Exchange hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] @@ -105,7 +105,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number# (8) CometExchange Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), Exchange hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [2]: [sr_item_sk#9, sr_ticket_number#10] @@ -149,7 +149,7 @@ Results [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] (18) Exchange Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] (19) HashAggregate [codegen id : 3] Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#17, sum#18, sum#19] @@ -176,7 +176,7 @@ Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) (23) CometExchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), Exchange hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=6], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (24) CometSort Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] @@ -199,7 +199,7 @@ Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number# (28) CometExchange Input [2]: [wr_item_sk#35, wr_order_number#36] -Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), Exchange hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, [plan_id=8], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (29) CometSort Input [2]: [wr_item_sk#35, wr_order_number#36] @@ -243,7 +243,7 @@ Results [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, (38) Exchange Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=6] (39) HashAggregate [codegen id : 6] Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] @@ -284,7 +284,7 @@ Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) (46) CometExchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), Exchange hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (47) CometSort Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] @@ -307,7 +307,7 @@ Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number# (51) CometExchange Input [2]: [cr_item_sk#62, cr_order_number#63] -Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), Exchange hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, [plan_id=13], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (52) CometSort Input [2]: [cr_item_sk#62, cr_order_number#63] @@ -351,7 +351,7 @@ Results [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, (61) Exchange Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=15] +Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=9] (62) HashAggregate [codegen id : 10] Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] @@ -407,7 +407,7 @@ Input [2]: [d_date_sk#12, d_year#13] (71) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] Subquery:2 Hosting operator id = 21 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt index d9d9be3c1f..dfd081b442 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt @@ -132,7 +132,7 @@ Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnul (3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), Exchange hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] @@ -155,7 +155,7 @@ Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] (8) CometExchange Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), Exchange hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=3], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] @@ -202,7 +202,7 @@ Input [2]: [s_store_sk#15, s_store_id#16] (19) BroadcastExchange Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] (20) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#2] @@ -234,7 +234,7 @@ Input [1]: [i_item_sk#17] (26) BroadcastExchange Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (27) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#1] @@ -266,7 +266,7 @@ Input [1]: [p_promo_sk#19] (33) BroadcastExchange Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_promo_sk#3] @@ -287,7 +287,7 @@ Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] (37) Exchange Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=6] (38) HashAggregate [codegen id : 6] Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] @@ -310,7 +310,7 @@ Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND (41) CometExchange Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), Exchange hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=9], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (42) CometSort Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] @@ -333,7 +333,7 @@ Arguments: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss# (46) CometExchange Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] -Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), Exchange hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, [plan_id=11], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Arguments: hashpartitioning(cr_item_sk#47, cr_order_number#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (47) CometSort Input [4]: [cr_item_sk#47, cr_order_number#48, cr_return_amount#49, cr_net_loss#50] @@ -380,7 +380,7 @@ Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] (57) BroadcastExchange Input [2]: [cp_catalog_page_sk#53, cp_catalog_page_id#54] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] (58) BroadcastHashJoin [codegen id : 11] Left keys [1]: [cs_catalog_page_sk#39] @@ -427,7 +427,7 @@ Results [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty (67) Exchange Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] -Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=14] +Arguments: hashpartitioning(cp_catalog_page_id#54, 5), ENSURE_REQUIREMENTS, [plan_id=10] (68) HashAggregate [codegen id : 12] Input [6]: [cp_catalog_page_id#54, sum#62, sum#63, isEmpty#64, sum#65, isEmpty#66] @@ -450,7 +450,7 @@ Condition : ((isnotnull(ws_web_site_sk#76) AND isnotnull(ws_item_sk#75)) AND isn (71) CometExchange Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), Exchange hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, [plan_id=15], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=16] +Arguments: hashpartitioning(ws_item_sk#75, ws_order_number#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (72) CometSort Input [7]: [ws_item_sk#75, ws_web_site_sk#76, ws_promo_sk#77, ws_order_number#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] @@ -473,7 +473,7 @@ Arguments: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] (76) CometExchange Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] -Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), Exchange hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, [plan_id=17], ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] +Arguments: hashpartitioning(wr_item_sk#83, wr_order_number#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (77) CometSort Input [4]: [wr_item_sk#83, wr_order_number#84, wr_return_amt#85, wr_net_loss#86] @@ -520,7 +520,7 @@ Input [2]: [web_site_sk#89, web_site_id#90] (87) BroadcastExchange Input [2]: [web_site_sk#89, web_site_id#90] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] (88) BroadcastHashJoin [codegen id : 17] Left keys [1]: [ws_web_site_sk#76] @@ -567,7 +567,7 @@ Results [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (97) Exchange Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=20] +Arguments: hashpartitioning(web_site_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=14] (98) HashAggregate [codegen id : 18] Input [6]: [web_site_id#90, sum#98, sum#99, isEmpty#100, sum#101, isEmpty#102] @@ -587,7 +587,7 @@ Results [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum (101) Exchange Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=21] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=15] (102) HashAggregate [codegen id : 20] Input [8]: [channel#34, id#35, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] @@ -615,7 +615,7 @@ Results [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, i (106) Exchange Input [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=22] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=16] (107) HashAggregate [codegen id : 41] Input [7]: [channel#34, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] @@ -643,7 +643,7 @@ Results [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] (111) Exchange Input [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=17] (112) HashAggregate [codegen id : 62] Input [6]: [sum#157, isEmpty#158, sum#159, isEmpty#160, sum#161, isEmpty#162] @@ -663,7 +663,7 @@ Results [5]: [channel#34, id#35, sales#126, returns#127, profit#128] (115) Exchange Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] -Arguments: hashpartitioning(channel#34, id#35, sales#126, returns#127, profit#128, 5), ENSURE_REQUIREMENTS, [plan_id=24] +Arguments: hashpartitioning(channel#34, id#35, sales#126, returns#127, profit#128, 5), ENSURE_REQUIREMENTS, [plan_id=18] (116) HashAggregate [codegen id : 64] Input [5]: [channel#34, id#35, sales#126, returns#127, profit#128] @@ -706,7 +706,7 @@ Input [1]: [d_date_sk#14] (122) BroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=19] Subquery:2 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#45 IN dynamicpruning#8 From f6e9dc3e5a93b9f89ad5da6f9b1d48e15805e0b8 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 25 Apr 2024 14:26:42 -0700 Subject: [PATCH 5/8] Dedup test --- .../apache/comet/exec/CometExecSuite.scala | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 383eca8ec6..2775efee14 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.Hex import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateMode import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometCollectLimitExec, CometFilterExec, CometHashAggregateExec, CometHashJoinExec, CometProjectExec, CometRowToColumnarExec, CometScanExec, CometSortExec, CometSortMergeJoinExec, CometTakeOrderedAndProjectExec} import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometShuffleExchangeExec} -import org.apache.spark.sql.execution.{CollectLimitExec, ProjectExec, SQLExecution, UnionExec} +import org.apache.spark.sql.execution.{CollectLimitExec, ProjectExec, SparkPlan, SQLExecution, UnionExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{BroadcastNestedLoopJoinExec, CartesianProductExec, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec @@ -69,26 +69,18 @@ class CometExecSuite extends CometTestBase { .toDF("c1", "c2") .createOrReplaceTempView("v") - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key -> "true") { - val df = sql("SELECT * FROM v where c1 = 1 order by c1, c2") - val shuffle = find(df.queryExecution.executedPlan) { - case _: CometShuffleExchangeExec => true - case _ => false - }.get.asInstanceOf[CometShuffleExchangeExec] - assert(shuffle.logicalLink.isEmpty) - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key -> "false") { - val df = sql("SELECT * FROM v where c1 = 1 order by c1, c2") - val shuffle = find(df.queryExecution.executedPlan) { - case _: ShuffleExchangeExec => true - case _ => false - }.get.asInstanceOf[ShuffleExchangeExec] - assert(shuffle.logicalLink.isEmpty) + Seq(true, false).foreach { columnarShuffle => + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key -> columnarShuffle.toString) { + val df = sql("SELECT * FROM v where c1 = 1 order by c1, c2") + val shuffle = find(df.queryExecution.executedPlan) { + case _: CometShuffleExchangeExec if columnarShuffle => true + case _: ShuffleExchangeExec if !columnarShuffle => true + case _ => false + }.get + assert(shuffle.logicalLink.isEmpty) + } } } } From 87f50dc5ad436e6dbd1f4d53f1a4f307421b4221 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 25 Apr 2024 17:02:28 -0700 Subject: [PATCH 6/8] Remove unused import --- spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 2775efee14..25a5fe72d5 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.Hex import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateMode import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometCollectLimitExec, CometFilterExec, CometHashAggregateExec, CometHashJoinExec, CometProjectExec, CometRowToColumnarExec, CometScanExec, CometSortExec, CometSortMergeJoinExec, CometTakeOrderedAndProjectExec} import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometShuffleExchangeExec} -import org.apache.spark.sql.execution.{CollectLimitExec, ProjectExec, SparkPlan, SQLExecution, UnionExec} +import org.apache.spark.sql.execution.{CollectLimitExec, ProjectExec, SQLExecution, UnionExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{BroadcastNestedLoopJoinExec, CartesianProductExec, SortMergeJoinExec} import org.apache.spark.sql.execution.window.WindowExec From cf7b38209713519a3686c3ad8b9e30d32101b662 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 25 Apr 2024 17:58:55 -0700 Subject: [PATCH 7/8] Fix test --- .../main/scala/org/apache/spark/sql/comet/operators.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index a8579757d5..adbe412de7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -239,6 +239,7 @@ abstract class CometNativeExec extends CometExec { val firstNonBroadcastPlan = sparkPlans.zipWithIndex.find { case (_: CometBroadcastExchangeExec, _) => false case (BroadcastQueryStageExec(_, _: CometBroadcastExchangeExec, _), _) => false + case (BroadcastQueryStageExec(_, _: ReusedExchangeExec, _), _) => false case _ => true } @@ -263,6 +264,13 @@ abstract class CometNativeExec extends CometExec { inputs += c.setNumPartitions(firstNonBroadcastPlanNumPartitions).executeColumnar() case BroadcastQueryStageExec(_, c: CometBroadcastExchangeExec, _) => inputs += c.setNumPartitions(firstNonBroadcastPlanNumPartitions).executeColumnar() + case ReusedExchangeExec(_, c: CometBroadcastExchangeExec) => + inputs += c.setNumPartitions(firstNonBroadcastPlanNumPartitions).executeColumnar() + case BroadcastQueryStageExec( + _, + ReusedExchangeExec(_, c: CometBroadcastExchangeExec), + _) => + inputs += c.setNumPartitions(firstNonBroadcastPlanNumPartitions).executeColumnar() case _ if idx == firstNonBroadcastPlan.get._2 => inputs += firstNonBroadcastPlanRDD case _ => From f5e343765da29afedb484b3bb4636b546f5cd086 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 26 Apr 2024 22:46:31 -0700 Subject: [PATCH 8/8] Use columnar shuffle --- .../test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala index 6ec25dd19b..8d7111e8f9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala @@ -91,6 +91,7 @@ class CometTPCHQuerySuite extends QueryTest with CometTPCBase with SQLQueryTestH conf.set(CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_ALL_EXPR_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") + conf.set(CometConf.COMET_COLUMNAR_SHUFFLE_ENABLED.key, "true") conf.set(MEMORY_OFFHEAP_ENABLED.key, "true") conf.set(MEMORY_OFFHEAP_SIZE.key, "2g") }