diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index 507198d2dd..00313205be 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -108,7 +108,7 @@ Note that the output files get written to `$SPARK_HOME`. The tests can be run with: ```sh -export SPARK_HOME=`pwd` COMET_PARQUET_SCAN_IMPL=native_comet +export SPARK_HOME=`pwd` ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.4 -nsu test ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.5 -nsu test ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-4.0 -nsu test @@ -116,7 +116,7 @@ export SPARK_HOME=`pwd` COMET_PARQUET_SCAN_IMPL=native_comet and ```sh -export SPARK_HOME=`pwd` COMET_PARQUET_SCAN_IMPL=native_comet +export SPARK_HOME=`pwd` ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.4 -nsu test ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.5 -nsu test ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-4.0 -nsu test @@ -126,7 +126,7 @@ If your pull request changes the query plans generated by Comet, you should rege To regenerate the golden files, you can run the following commands. ```sh -export SPARK_HOME=`pwd` COMET_PARQUET_SCAN_IMPL=native_comet +export SPARK_HOME=`pwd` SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.4 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.5 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-4.0 -nsu test @@ -134,8 +134,8 @@ SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.co and ```sh -export SPARK_HOME=`pwd` COMET_PARQUET_SCAN_IMPL=native_comet -SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -nsu test +export SPARK_HOME=`pwd` +SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.4 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.5 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-4.0 -nsu test ``` diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt index 8c3bf99beb..e66401996f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt @@ -14,11 +14,11 @@ : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (25) : : +- CometFilter (24) : : +- CometHashAggregate (23) @@ -30,19 +30,19 @@ : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (13) : : +- ReusedExchange (15) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_comet] parquet spark_catalog.default.store (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.customer (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) -(1) CometScan [native_comet] parquet spark_catalog.default.store_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -54,7 +54,7 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_county#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#4] Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt index dcab21d624..e7193f87e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt index 8dd34e58ed..091ad62567 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt @@ -17,13 +17,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -34,13 +34,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (52) : +- CometFilter (51) : +- CometHashAggregate (50) @@ -52,10 +52,10 @@ : : +- CometBroadcastHashJoin (43) : : :- CometProject (39) : : : +- CometFilter (38) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) : : +- CometBroadcastExchange (42) : : +- CometFilter (41) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : +- ReusedExchange (45) +- CometBroadcastExchange (69) +- CometHashAggregate (68) @@ -67,14 +67,14 @@ : +- CometBroadcastHashJoin (61) : :- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (55) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) : +- CometBroadcastExchange (60) : +- CometFilter (59) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (58) +- ReusedExchange (63) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -89,7 +89,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -114,7 +114,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -156,7 +156,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -171,7 +171,7 @@ Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Batched: true Location: InMemoryFileIndex [] @@ -196,7 +196,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -247,7 +247,7 @@ Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight Input [5]: [customer_id#23, year_total#24, customer_id#41, customer_preferred_cust_flag#42, year_total#43] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] -(37) CometScan [native_comet] parquet spark_catalog.default.customer +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -262,7 +262,7 @@ Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] -(40) CometScan [native_comet] parquet spark_catalog.default.web_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] Batched: true Location: InMemoryFileIndex [] @@ -330,7 +330,7 @@ Arguments: [customer_id#23], [customer_id#66], Inner, BuildRight Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, customer_id#66, year_total#67] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] -(55) CometScan [native_comet] parquet spark_catalog.default.customer +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -345,7 +345,7 @@ Condition : (isnotnull(c_customer_sk#68) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] Arguments: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57], [c_customer_sk#68, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#69, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#70, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#71, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#72, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#73, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#74, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#75, 50, true, false, true) AS c_email_address#57] -(58) CometScan [native_comet] parquet spark_catalog.default.web_sales +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] @@ -422,10 +422,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometFilter (75) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) -(74) CometScan [native_comet] parquet spark_catalog.default.date_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -447,10 +447,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometFilter (79) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (78) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (78) -(78) CometScan [native_comet] parquet spark_catalog.default.date_dim +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt index 131ebf2656..a3bacdf562 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt @@ -18,20 +18,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 CometHashAggregate [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -42,20 +42,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -67,10 +67,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -83,9 +83,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt index 88341ba737..93308784a9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt @@ -13,18 +13,18 @@ TakeOrderedAndProject (23) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt index f48019d841..0a1a7d8020 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt index 9b7dbcb6e6..74da8ba883 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt @@ -14,28 +14,28 @@ : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) +- CometBroadcastExchange (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -47,7 +47,7 @@ ReadSchema: struct= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometScan [native_comet] parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -71,7 +71,7 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -99,7 +99,7 @@ Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -155,7 +155,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -(26) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#24, hd_dep_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -203,10 +203,10 @@ BroadcastExchange (39) +- * CometColumnarToRow (38) +- CometProject (37) +- CometFilter (36) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) -(35) CometScan [native_comet] parquet spark_catalog.default.date_dim +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt index 2511aa3485..a1a0200121 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -23,22 +23,22 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt index 1870a87a01..22f1896b63 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt @@ -17,12 +17,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -33,25 +33,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -59,18 +59,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) :- CometProject (80) : +- CometFilter (79) : +- CometHashAggregate (78) @@ -82,7 +82,7 @@ : : +- CometBroadcastHashJoin (71) : : :- CometBroadcastHashJoin (69) : : : :- CometFilter (67) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (66) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (66) : : : +- ReusedExchange (68) : : +- ReusedExchange (70) : +- ReusedExchange (73) @@ -97,13 +97,13 @@ : +- CometBroadcastHashJoin (86) : :- CometBroadcastHashJoin (84) : : :- CometFilter (82) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (81) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (81) : : +- ReusedExchange (83) : +- ReusedExchange (85) +- ReusedExchange (88) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -126,7 +126,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -149,7 +149,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -185,7 +185,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -261,7 +261,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -328,7 +328,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -360,7 +360,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,7 +410,7 @@ Condition : (isnotnull(sales#49) AND (cast(sales#49 as decimal(32,6)) > cast(Sub Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] Arguments: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56], [sales#49, number_sales#50, store AS channel#53, i_brand_id#40 AS i_brand_id#54, i_class_id#41 AS i_class_id#55, i_category_id#42 AS i_category_id#56] -(66) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] @@ -476,7 +476,7 @@ Condition : (isnotnull(sales#71) AND (cast(sales#71 as decimal(32,6)) > cast(Reu Input [5]: [i_brand_id#64, i_class_id#65, i_category_id#66, sales#71, number_sales#72] Arguments: [sales#71, number_sales#72, channel#73, i_brand_id#64, i_class_id#65, i_category_id#66], [sales#71, number_sales#72, catalog AS channel#73, i_brand_id#64, i_class_id#65, i_category_id#66] -(81) CometScan [native_comet] parquet spark_catalog.default.web_sales +(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] @@ -582,19 +582,19 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (115) :- CometProject (106) : +- CometBroadcastHashJoin (105) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (103) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (103) : +- ReusedExchange (104) :- CometProject (110) : +- CometBroadcastHashJoin (109) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (107) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (107) : +- ReusedExchange (108) +- CometProject (114) +- CometBroadcastHashJoin (113) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (111) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (111) +- ReusedExchange (112) -(103) CometScan [native_comet] parquet spark_catalog.default.store_sales +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103] Batched: true Location: InMemoryFileIndex [] @@ -613,7 +613,7 @@ Arguments: [ss_sold_date_sk#103], [d_date_sk#105], Inner, BuildRight Input [4]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103, d_date_sk#105] Arguments: [quantity#106, list_price#107], [ss_quantity#101 AS quantity#106, ss_list_price#102 AS list_price#107] -(107) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110] Batched: true Location: InMemoryFileIndex [] @@ -632,7 +632,7 @@ Arguments: [cs_sold_date_sk#110], [d_date_sk#112], Inner, BuildRight Input [4]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110, d_date_sk#112] Arguments: [quantity#113, list_price#114], [cs_quantity#108 AS quantity#113, cs_list_price#109 AS list_price#114] -(111) CometScan [native_comet] parquet spark_catalog.default.web_sales +(111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#115, ws_list_price#116, ws_sold_date_sk#117] Batched: true Location: InMemoryFileIndex [] @@ -684,10 +684,10 @@ BroadcastExchange (124) +- * CometColumnarToRow (123) +- CometProject (122) +- CometFilter (121) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (120) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (120) -(120) CometScan [native_comet] parquet spark_catalog.default.date_dim +(120) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -714,10 +714,10 @@ BroadcastExchange (129) +- * CometColumnarToRow (128) +- CometProject (127) +- CometFilter (126) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (125) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) -(125) CometScan [native_comet] parquet spark_catalog.default.date_dim +(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#125] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt index 730f748777..13c79b4299 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt @@ -19,17 +19,17 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -49,12 +49,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -65,7 +65,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) @@ -73,26 +73,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 CometProject [i_brand_id,i_class_id,i_category_id] @@ -100,19 +100,19 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #4 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 @@ -125,7 +125,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 @@ -142,7 +142,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt index feb64d0cec..879213d892 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt @@ -12,12 +12,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -28,25 +28,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -54,18 +54,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -77,16 +77,16 @@ : +- CometBroadcastHashJoin (70) : :- CometBroadcastHashJoin (68) : : :- CometFilter (66) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) : : +- ReusedExchange (67) : +- ReusedExchange (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -109,7 +109,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -132,7 +132,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -244,7 +244,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -389,7 +389,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) -(65) CometScan [native_comet] parquet spark_catalog.default.store_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -(72) CometScan [native_comet] parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -493,19 +493,19 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (98) :- CometProject (89) : +- CometBroadcastHashJoin (88) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (86) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) : +- ReusedExchange (87) :- CometProject (93) : +- CometBroadcastHashJoin (92) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (90) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) : +- ReusedExchange (91) +- CometProject (97) +- CometBroadcastHashJoin (96) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (94) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) +- ReusedExchange (95) -(86) CometScan [native_comet] parquet spark_catalog.default.store_sales +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] @@ -524,7 +524,7 @@ Arguments: [ss_sold_date_sk#77], [d_date_sk#79], Inner, BuildRight Input [4]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77, d_date_sk#79] Arguments: [quantity#80, list_price#81], [ss_quantity#75 AS quantity#80, ss_list_price#76 AS list_price#81] -(90) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -543,7 +543,7 @@ Arguments: [cs_sold_date_sk#84], [d_date_sk#86], Inner, BuildRight Input [4]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84, d_date_sk#86] Arguments: [quantity#87, list_price#88], [cs_quantity#82 AS quantity#87, cs_list_price#83 AS list_price#88] -(94) CometScan [native_comet] parquet spark_catalog.default.web_sales +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] Batched: true Location: InMemoryFileIndex [] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (103) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_comet] parquet spark_catalog.default.date_dim +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_comet] parquet spark_catalog.default.date_dim +(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#103] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (117) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_comet] parquet spark_catalog.default.date_dim +(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (122) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_comet] parquet spark_catalog.default.date_dim +(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt index 0e0b2ea4b8..5d5557bc59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) @@ -75,26 +75,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -102,20 +102,20 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -128,7 +128,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #17 WholeStageCodegen (1) @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt index d6b99398ca..af9b2efbd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt @@ -11,21 +11,21 @@ : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) +- CometBroadcastExchange (17) +- CometProject (16) +- CometFilter (15) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.customer +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#5, c_current_addr_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -61,7 +61,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -89,7 +89,7 @@ Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_comet] parquet spark_catalog.default.date_dim +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt index 6105b85eb5..6eac0d3bd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt index a7a73163fd..1654e70a54 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (12) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (23) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.call_center (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -92,7 +92,7 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -115,7 +115,7 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometScan [native_comet] parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometScan [native_comet] parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometScan [native_comet] parquet spark_catalog.default.call_center +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt index 96c978299d..352d9e22b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [cs_order_number] #2 CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] CometSort [cs_warehouse_sk,cs_order_number] CometExchange [cs_order_number] #3 CometProject [cs_warehouse_sk,cs_order_number] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] CometSort [cr_order_number] CometExchange [cr_order_number] #4 CometProject [cr_order_number] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cc_call_center_sk] #7 CometProject [cc_call_center_sk] CometFilter [cc_call_center_sk,cc_county] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt index d37f4a2a69..f502a5cd8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt @@ -19,33 +19,33 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_comet] parquet spark_catalog.default.store (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.item (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -57,7 +57,7 @@ ReadSchema: struct Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometScan [native_comet] parquet spark_catalog.default.date_dim +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#1 Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(15) CometScan [native_comet] parquet spark_catalog.default.date_dim +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -139,7 +139,7 @@ Input [8]: [d_week_seq#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#4 Keys [1]: [d_week_seq#37] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(23) CometScan [native_comet] parquet spark_catalog.default.date_dim +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#45, d_year#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt index 9d342c2218..25ed325468 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] CometUnion [sold_date_sk,sales_price] CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] @@ -33,4 +33,4 @@ WholeStageCodegen (1) CometBroadcastExchange [d_week_seq] #6 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt index 51fba80c47..c23383e09a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt @@ -13,18 +13,18 @@ TakeOrderedAndProject (23) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt index 67283a9aba..fdf7f7ffa6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt index 83d5f82979..5656916223 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt @@ -12,20 +12,20 @@ : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_comet] parquet spark_catalog.default.item (8) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +37,7 @@ ReadSchema: struct -(2) CometScan [native_comet] parquet spark_catalog.default.store_sales +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -87,7 +87,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -115,7 +115,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(10) CometScan [native_comet] parquet spark_catalog.default.item +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -182,7 +182,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(25) CometScan [native_comet] parquet spark_catalog.default.store_sales +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -197,7 +197,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(28) CometScan [native_comet] parquet spark_catalog.default.customer +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,7 +256,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -284,7 +284,7 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#29], Inner, BuildRight Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#29] Arguments: [sales#32], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#32] -(47) CometScan [native_comet] parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] Batched: true Location: InMemoryFileIndex [] @@ -380,10 +380,10 @@ BroadcastExchange (72) +- * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_comet] parquet spark_catalog.default.date_dim +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,10 +410,10 @@ BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometProject (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -448,15 +448,15 @@ Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquer :- CometProject (82) : +- CometBroadcastHashJoin (81) : :- CometFilter (79) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (78) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (78) : +- ReusedExchange (80) +- CometBroadcastExchange (86) +- CometProject (85) +- CometFilter (84) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (83) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (83) -(78) CometScan [native_comet] parquet spark_catalog.default.store_sales +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -480,7 +480,7 @@ Arguments: [ss_customer_sk#51], [c_customer_sk#56], Inner, BuildRight Input [5]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] Arguments: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56], [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] -(83) CometScan [native_comet] parquet spark_catalog.default.date_dim +(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#57, d_year#58] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -544,10 +544,10 @@ BroadcastExchange (100) +- * CometColumnarToRow (99) +- CometProject (98) +- CometFilter (97) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (96) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (96) -(96) CometScan [native_comet] parquet spark_catalog.default.date_dim +(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#57, d_year#58] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt index 7fed1a3233..cd74cced17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -21,7 +21,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -33,7 +33,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) @@ -41,14 +41,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #13 WholeStageCodegen (1) @@ -75,12 +75,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] @@ -88,14 +88,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_quantity,ws_list_price] [sales] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] @@ -104,7 +104,7 @@ WholeStageCodegen (1) CometExchange [ws_bill_customer_sk] #16 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [item_sk] #4 CometSort [c_customer_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt index 372bd21d93..474a708a44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt @@ -15,7 +15,7 @@ : : : : +- CometProject (23) : : : : +- CometBroadcastHashJoin (22) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (21) : : : : +- CometProject (20) : : : : +- CometFilter (19) @@ -27,14 +27,14 @@ : : : : :- CometProject (10) : : : : : +- CometBroadcastHashJoin (9) : : : : : :- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : : +- CometBroadcastExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) : : : : +- CometBroadcastExchange (13) : : : : +- CometFilter (12) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : +- CometSort (39) : : : +- CometProject (38) : : : +- CometFilter (37) @@ -45,17 +45,17 @@ : : : +- CometBroadcastHashJoin (32) : : : :- CometProject (28) : : : : +- CometFilter (27) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (26) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) : : : +- CometBroadcastExchange (31) : : : +- CometFilter (30) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (29) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) : : +- CometBroadcastExchange (52) : : +- CometProject (51) : : +- CometSortMergeJoin (50) : : :- CometSort (44) : : : +- CometExchange (43) : : : +- CometFilter (42) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (41) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) : : +- CometSort (49) : : +- CometProject (48) : : +- CometFilter (47) @@ -64,7 +64,7 @@ : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometHashAggregate (85) +- CometExchange (84) +- CometHashAggregate (83) @@ -78,7 +78,7 @@ : : : +- CometProject (68) : : : +- CometBroadcastHashJoin (67) : : : :- CometFilter (65) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (64) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (64) : : : +- ReusedExchange (66) : : +- CometSort (75) : : +- CometProject (74) @@ -89,7 +89,7 @@ +- ReusedExchange (80) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -101,7 +101,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(5) CometScan [native_comet] parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -141,7 +141,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -208,7 +208,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(26) CometScan [native_comet] parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -223,7 +223,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(29) CometScan [native_comet] parquet spark_catalog.default.customer +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -278,7 +278,7 @@ Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold Right output [1]: [c_customer_sk#23] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi -(41) CometScan [native_comet] parquet spark_catalog.default.customer +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -339,7 +339,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#29], Inner, BuildRight Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#29, c_first_name#32, c_last_name#33] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -381,7 +381,7 @@ Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] Keys [2]: [c_last_name#33, c_first_name#32] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(64) CometScan [native_comet] parquet spark_catalog.default.web_sales +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -494,10 +494,10 @@ BroadcastExchange (93) +- * CometColumnarToRow (92) +- CometProject (91) +- CometFilter (90) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (89) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (89) -(89) CometScan [native_comet] parquet spark_catalog.default.date_dim +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -524,10 +524,10 @@ BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometProject (96) +- CometFilter (95) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (94) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) -(94) CometScan [native_comet] parquet spark_catalog.default.date_dim +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -562,15 +562,15 @@ Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquer :- CometProject (103) : +- CometBroadcastHashJoin (102) : :- CometFilter (100) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (99) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (99) : +- ReusedExchange (101) +- CometBroadcastExchange (107) +- CometProject (106) +- CometFilter (105) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) -(99) CometScan [native_comet] parquet spark_catalog.default.store_sales +(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] @@ -594,7 +594,7 @@ Arguments: [ss_customer_sk#60], [c_customer_sk#65], Inner, BuildRight Input [5]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] Arguments: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65], [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] -(104) CometScan [native_comet] parquet spark_catalog.default.date_dim +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#66, d_year#67] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -658,10 +658,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (117) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_comet] parquet spark_catalog.default.date_dim +(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#66, d_year#67] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt index 96a3fd3d50..0575c4a672 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,7 +24,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) @@ -44,14 +44,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -70,7 +70,7 @@ WholeStageCodegen (1) CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #13 WholeStageCodegen (1) @@ -78,12 +78,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] @@ -91,17 +91,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] CometExchange [c_customer_sk] #16 CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -111,7 +111,7 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #17 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] CometExchange [c_last_name,c_first_name] #18 CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,ws_quantity,ws_list_price] @@ -125,7 +125,7 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [item_sk] #4 CometSort [c_customer_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt index 37c38ba43d..c6e3a30a1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_comet] parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_comet] parquet spark_catalog.default.item +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_comet] parquet spark_catalog.default.customer_address +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_comet] parquet spark_catalog.default.item +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt index 8444341b4c..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt index 3a3034734d..a77a3f88b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_comet] parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_comet] parquet spark_catalog.default.item +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_comet] parquet spark_catalog.default.customer_address +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_comet] parquet spark_catalog.default.item +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt index 8444341b4c..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt index 2b4aac2e83..6a9f9094d3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt @@ -19,33 +19,33 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_comet] parquet spark_catalog.default.store (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.item (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -57,7 +57,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_county#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -165,7 +165,7 @@ Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] Keys [3]: [ca_county#9, d_qoy#7, d_year#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(16) CometScan [native_comet] parquet spark_catalog.default.store_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -177,7 +177,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] Condition : isnotnull(ss_addr_sk#11) -(18) CometScan [native_comet] parquet spark_catalog.default.date_dim +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -236,7 +236,7 @@ Left output [3]: [ca_county#9, d_year#6, store_sales#22] Right output [2]: [ca_county#19, store_sales#21] Arguments: [ca_county#9], [ca_county#19], Inner, BuildRight -(31) CometScan [native_comet] parquet spark_catalog.default.store_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -248,7 +248,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_addr_sk#23) -(33) CometScan [native_comet] parquet spark_catalog.default.date_dim +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -311,7 +311,7 @@ Arguments: [ca_county#19], [ca_county#31], Inner, BuildRight Input [7]: [ca_county#9, d_year#6, store_sales#22, ca_county#19, store_sales#21, ca_county#31, store_sales#33] Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33] -(47) CometScan [native_comet] parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] Batched: true Location: InMemoryFileIndex [] @@ -370,7 +370,7 @@ Left output [5]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_s Right output [2]: [ca_county#42, web_sales#44] Arguments: [ca_county#9], [ca_county#42], Inner, BuildRight -(60) CometScan [native_comet] parquet spark_catalog.default.web_sales +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -433,7 +433,7 @@ Arguments: [ca_county#42], [ca_county#53], Inner, (CASE WHEN (web_sales#44 > 0.0 Input [9]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, ca_county#53, web_sales#55] Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55] -(74) CometScan [native_comet] parquet spark_catalog.default.web_sales +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -513,10 +513,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dyn BroadcastExchange (94) +- * CometColumnarToRow (93) +- CometFilter (92) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (91) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (91) -(91) CometScan [native_comet] parquet spark_catalog.default.date_dim +(91) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -538,10 +538,10 @@ Subquery:2 Hosting operator id = 16 Hosting Expression = ss_sold_date_sk#13 IN d BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometFilter (96) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (95) -(95) CometScan [native_comet] parquet spark_catalog.default.date_dim +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -563,10 +563,10 @@ Subquery:3 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN d BroadcastExchange (102) +- * CometColumnarToRow (101) +- CometFilter (100) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (99) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (99) -(99) CometScan [native_comet] parquet spark_catalog.default.date_dim +(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt index 21535c42f5..435da991de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt @@ -19,20 +19,20 @@ WholeStageCodegen (1) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [ca_address_sk,ca_county] #5 CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #6 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #7 @@ -42,17 +42,17 @@ WholeStageCodegen (1) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,store_sales] #10 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] @@ -63,17 +63,17 @@ WholeStageCodegen (1) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #12 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #14 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] @@ -84,7 +84,7 @@ WholeStageCodegen (1) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year,d_qoy] #4 ReusedExchange [ca_address_sk,ca_county] #5 @@ -97,7 +97,7 @@ WholeStageCodegen (1) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #9 ReusedExchange [ca_address_sk,ca_county] #5 @@ -110,7 +110,7 @@ WholeStageCodegen (1) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk,d_year,d_qoy] #13 ReusedExchange [ca_address_sk,ca_county] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt index 0f119c6362..d6c10008ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,15 +23,15 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -71,7 +71,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -(9) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Condition : isnotnull(cs_item_sk#7) -(11) CometScan [native_comet] parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_comet] parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt index 6f783863c1..a89a381578 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt @@ -11,7 +11,7 @@ WholeStageCodegen (1) CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,sum,count,avg(UnscaledValue(cs_ext_discount_amt))] @@ -32,10 +32,10 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt index 7eb9890d8d..a178860ccc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt @@ -15,23 +15,23 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- CometHashAggregate (41) : +- CometExchange (40) : +- CometHashAggregate (39) @@ -42,7 +42,7 @@ : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometFilter (29) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (28) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (28) : : : +- ReusedExchange (30) : : +- ReusedExchange (33) : +- ReusedExchange (36) @@ -56,13 +56,13 @@ : :- CometProject (46) : : +- CometBroadcastHashJoin (45) : : :- CometFilter (43) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (42) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (42) : : +- ReusedExchange (44) : +- ReusedExchange (47) +- ReusedExchange (50) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -74,7 +74,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_manufact_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_category#13, i_manufact_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -191,7 +191,7 @@ Input [2]: [i_manufact_id#12, sum#15] Keys [1]: [i_manufact_id#12] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] @@ -253,7 +253,7 @@ Input [2]: [i_manufact_id#24, sum#25] Keys [1]: [i_manufact_id#24] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] -(42) CometScan [native_comet] parquet spark_catalog.default.web_sales +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] @@ -348,10 +348,10 @@ BroadcastExchange (66) +- * CometColumnarToRow (65) +- CometProject (64) +- CometFilter (63) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (62) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (62) -(62) CometScan [native_comet] parquet spark_catalog.default.date_dim +(62) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt index 9d856f475c..0628508d3f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,23 +24,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_manufact_id] #6 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #7 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_manufact_id] #8 CometHashAggregate [i_manufact_id,sum,cs_ext_sales_price] @@ -51,7 +51,7 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -66,7 +66,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt index b4dd66bb54..ceab9d3f0a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt @@ -15,26 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt index 238061487f..a7c1215d1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt index 2acb8a0ec9..fedc50de1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt @@ -15,40 +15,40 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (13) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) : : : +- ReusedExchange (14) : : +- BroadcastExchange (25) : : +- * CometColumnarToRow (24) : : +- CometProject (23) : : +- CometBroadcastHashJoin (22) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (20) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) : : +- ReusedExchange (21) : +- BroadcastExchange (33) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,14 +59,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#22] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt index 0d595eb78c..f1fe09fb46 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt index 075857a6b9..ae1d4c242b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt @@ -16,22 +16,22 @@ TakeOrderedAndProject (30) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) +- CometBroadcastExchange (18) +- CometProject (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct= 68.00)) A Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan [native_comet] parquet spark_catalog.default.inventory +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +71,7 @@ Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#13, cs_sold_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt index 660b5c5de8..7ab646a628 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt @@ -14,11 +14,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -26,11 +26,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [cs_item_sk] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt index bb790ceb71..591fa114fb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt @@ -14,15 +14,15 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : +- CometBroadcastExchange (29) : +- CometHashAggregate (28) : +- CometExchange (27) @@ -32,7 +32,7 @@ : :- CometProject (22) : : +- CometBroadcastHashJoin (21) : : :- CometFilter (19) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- ReusedExchange (23) +- CometBroadcastExchange (42) @@ -44,12 +44,12 @@ :- CometProject (35) : +- CometBroadcastHashJoin (34) : :- CometFilter (32) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (31) : +- ReusedExchange (33) +- ReusedExchange (36) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -89,7 +89,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_comet] parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -131,7 +131,7 @@ Input [3]: [c_last_name#11, c_first_name#10, d_date#5] Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] Functions: [] -(18) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -190,7 +190,7 @@ Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] Right output [3]: [c_last_name#19, c_first_name#18, d_date#16] Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)], LeftSemi, BuildRight -(31) CometScan [native_comet] parquet spark_catalog.default.web_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -276,10 +276,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt index fb925a86b6..c9ecc235c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -23,15 +23,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometExchange [c_last_name,c_first_name,d_date] #7 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 @@ -54,7 +54,7 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt index 6321c844f4..8e6d9265d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt @@ -15,17 +15,17 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -39,16 +39,16 @@ : :- CometProject (28) : : +- CometBroadcastHashJoin (27) : : :- CometFilter (25) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (24) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (26) : +- ReusedExchange (29) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ ReadSchema: struct 0.000000)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -205,7 +205,7 @@ Condition : (isnotnull(c_customer_sk#28) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] Arguments: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14], [c_customer_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#32, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#34, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#35, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -230,7 +230,7 @@ Arguments: [c_customer_sk#28], [ss_customer_sk#36], Inner, BuildRight Input [14]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -277,7 +277,7 @@ Left output [2]: [customer_id#26, year_total#27] Right output [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] Arguments: [customer_id#26], [customer_id#47], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -292,7 +292,7 @@ Condition : (isnotnull(c_customer_sk#55) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Arguments: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68], [c_customer_sk#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#56, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#57, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#58, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#59, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#61, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#62, 50, true, false, true) AS c_email_address#68] -(39) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -360,7 +360,7 @@ Arguments: [customer_id#26], [customer_id#80], Inner, BuildRight Input [12]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, customer_id#80, year_total#81] Arguments: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81], [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -375,7 +375,7 @@ Condition : (isnotnull(c_customer_sk#82) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] Arguments: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68], [c_customer_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#83, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#84, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#85, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#86, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#87, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#88, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#89, 50, true, false, true) AS c_email_address#68] -(57) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] Batched: true Location: InMemoryFileIndex [] @@ -439,7 +439,7 @@ Arguments: [customer_id#26], [customer_id#101], Inner, (CASE WHEN (year_total#81 Input [13]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81, customer_id#101, year_total#102] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] -(71) CometScan [native_comet] parquet spark_catalog.default.customer +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -454,7 +454,7 @@ Condition : (isnotnull(c_customer_sk#103) AND isnotnull(staticinvoke(class org.a Input [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] Arguments: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116], [c_customer_sk#103, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#104, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#105, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#106, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#107, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#109, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#110, 50, true, false, true) AS c_email_address#116] -(74) CometScan [native_comet] parquet spark_catalog.default.web_sales +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] Batched: true Location: InMemoryFileIndex [] @@ -522,7 +522,7 @@ Arguments: [customer_id#26], [customer_id#128], Inner, BuildRight Input [12]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, customer_id#128, year_total#129] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] -(89) CometScan [native_comet] parquet spark_catalog.default.customer +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -537,7 +537,7 @@ Condition : (isnotnull(c_customer_sk#130) AND isnotnull(staticinvoke(class org.a Input [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] Arguments: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116], [c_customer_sk#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#131, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#132, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#133, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#134, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#135, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#136, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#137, 50, true, false, true) AS c_email_address#116] -(92) CometScan [native_comet] parquet spark_catalog.default.web_sales +(92) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] Batched: true Location: InMemoryFileIndex [] @@ -614,10 +614,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dy BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometFilter (109) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_comet] parquet spark_catalog.default.date_dim +(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -639,10 +639,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#41 IN d BroadcastExchange (115) +- * CometColumnarToRow (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt index 8e74ad0f85..47e4022f45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt @@ -21,20 +21,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 @@ -45,20 +45,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] @@ -70,10 +70,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -86,10 +86,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 CometBroadcastExchange [customer_id,year_total] #16 @@ -103,10 +103,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #19 @@ -119,9 +119,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt index c83736dfb2..00e2b8fbc1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt @@ -15,26 +15,26 @@ : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_comet] parquet spark_catalog.default.item (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) +- CometBroadcastExchange (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -54,7 +54,7 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -86,7 +86,7 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -(12) CometScan [native_comet] parquet spark_catalog.default.warehouse +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#11, w_state#12] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -114,7 +114,7 @@ Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -(18) CometScan [native_comet] parquet spark_catalog.default.item +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -142,7 +142,7 @@ Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -193,10 +193,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dyn BroadcastExchange (37) +- * CometColumnarToRow (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt index 56f45f3550..23945dce83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt @@ -16,27 +16,27 @@ WholeStageCodegen (1) CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] CometExchange [cr_order_number,cr_item_sk] #4 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometBroadcastExchange [w_warehouse_sk,w_state] #5 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #7 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt index ec713c7076..568da58fa0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,10 +17,10 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometScan [native_comet] parquet spark_catalog.default.item (4) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -35,7 +35,7 @@ Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_ma Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometScan [native_comet] parquet spark_catalog.default.item +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt index b568d3ebc1..bff847188c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] @@ -18,4 +18,4 @@ WholeStageCodegen (1) CometHashAggregate [i_manufact,count] CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometScan [native_comet] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt index b709094800..183fffdf00 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt index d8fb182c5c..9de0f79dee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 CometProject [i_category] [i_item_sk,i_category_id,i_category] CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt index 0e66c78acb..62ddd93781 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.store (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometScan [native_comet] parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt index 738d423a98..c7445ea60c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [d_day_name] [d_date_sk,d_day_name] CometFilter [d_date_sk,d_year,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt index 0ab5998910..4b30c7d51f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt @@ -23,7 +23,7 @@ TakeOrderedAndProject (45) : : : +- CometHashAggregate (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- * Sort (32) : : +- * Project (31) : : +- * Filter (30) @@ -42,11 +42,11 @@ TakeOrderedAndProject (45) : +- * CometColumnarToRow (38) : +- CometProject (37) : +- CometFilter (36) - : +- CometScan [native_comet] parquet spark_catalog.default.item (35) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (35) +- ReusedExchange (42) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -185,7 +185,7 @@ Join condition: None Output [3]: [item_sk#7, rnk#11, item_sk#16] Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] -(35) CometScan [native_comet] parquet spark_catalog.default.item +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_product_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -243,10 +243,10 @@ Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (46) -(46) CometScan [native_comet] parquet spark_catalog.default.store_sales +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt index 6a0cccc2c8..018f1919e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt @@ -34,13 +34,13 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] CometProject [ss_store_sk,ss_net_profit] CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] CometExchange [ss_item_sk] #2 CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] CometProject [ss_item_sk,ss_net_profit] CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (6) Sort [rnk] @@ -70,6 +70,6 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] InputAdapter CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt index b704b4b954..f6ab9a8d53 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt @@ -17,30 +17,30 @@ TakeOrderedAndProject (39) : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.item (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) +- BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.item (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -52,7 +52,7 @@ ReadSchema: struct= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometScan [native_comet] parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -66,7 +66,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -94,7 +94,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(14) CometScan [native_comet] parquet spark_catalog.default.customer_address +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -122,7 +122,7 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH, Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(20) CometScan [native_comet] parquet spark_catalog.default.date_dim +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -174,10 +174,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt index 408dad4982..b456a3d4a3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -21,19 +21,19 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt index 3f4034ad0f..338f5f4201 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt @@ -23,14 +23,14 @@ : : :- CometBroadcastExchange (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometProject (7) : : +- CometFilter (6) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) :- * Project (49) : +- * Filter (48) : +- Window (47) @@ -49,10 +49,10 @@ : : :- CometBroadcastExchange (30) : : : +- CometProject (29) : : : +- CometFilter (28) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (27) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) : : +- CometProject (33) : : +- CometFilter (32) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) : +- ReusedExchange (36) +- * Project (72) +- * Filter (71) @@ -72,14 +72,14 @@ : :- CometBroadcastExchange (53) : : +- CometProject (52) : : +- CometFilter (51) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (50) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) : +- CometProject (56) : +- CometFilter (55) - : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (54) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) +- ReusedExchange (59) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -99,7 +99,7 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometScan [native_comet] parquet spark_catalog.default.web_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -123,7 +123,7 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -196,7 +196,7 @@ Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -(27) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] @@ -216,7 +216,7 @@ Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, c Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -(31) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -297,7 +297,7 @@ Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -(50) CometScan [native_comet] parquet spark_catalog.default.store_sales +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -317,7 +317,7 @@ Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -(54) CometScan [native_comet] parquet spark_catalog.default.store_returns +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -430,10 +430,10 @@ BroadcastExchange (83) +- * CometColumnarToRow (82) +- CometProject (81) +- CometFilter (80) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (79) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) -(79) CometScan [native_comet] parquet spark_catalog.default.date_dim +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt index 5e8cf98267..43ce5f13af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt @@ -32,7 +32,7 @@ WholeStageCodegen (11) CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) @@ -40,14 +40,14 @@ WholeStageCodegen (11) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] WholeStageCodegen (6) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -72,11 +72,11 @@ WholeStageCodegen (11) CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] @@ -102,9 +102,9 @@ WholeStageCodegen (11) CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt index 07fbe00126..f2dce3f729 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt @@ -17,18 +17,18 @@ TakeOrderedAndProject (74) : : :- CometUnion (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.store (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) :- * HashAggregate (43) : +- * CometColumnarToRow (42) : +- CometExchange (41) @@ -40,15 +40,15 @@ TakeOrderedAndProject (74) : : :- CometUnion (30) : : : :- CometProject (26) : : : : +- CometFilter (25) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (24) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) : : : +- CometProject (29) : : : +- CometFilter (28) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (27) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) : : +- ReusedExchange (31) : +- CometBroadcastExchange (37) : +- CometProject (36) : +- CometFilter (35) - : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (34) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) +- * HashAggregate (67) +- * CometColumnarToRow (66) +- CometExchange (65) @@ -60,22 +60,22 @@ TakeOrderedAndProject (74) : :- CometUnion (54) : : :- CometProject (46) : : : +- CometFilter (45) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (44) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) : : +- CometProject (53) : : +- CometBroadcastHashJoin (52) : : :- CometBroadcastExchange (48) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (47) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) : : +- CometProject (51) : : +- CometFilter (50) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (49) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) : +- ReusedExchange (55) +- CometBroadcastExchange (61) +- CometProject (60) +- CometFilter (59) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (58) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -91,7 +91,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -111,7 +111,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_comet] parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -139,7 +139,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_comet] parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -186,7 +186,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt# Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#37, store channel AS channel#38, concat(store, s_store_id#26) AS id#39] -(24) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -202,7 +202,7 @@ Condition : isnotnull(cs_catalog_page_sk#40) Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] -(27) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -234,7 +234,7 @@ Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -(34) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -281,7 +281,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#75, catalog channel AS channel#76, concat(catalog_page, cp_catalog_page_id#64) AS id#77] -(44) CometScan [native_comet] parquet spark_catalog.default.web_sales +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Batched: true Location: InMemoryFileIndex [] @@ -297,7 +297,7 @@ Condition : isnotnull(ws_web_site_sk#78) Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] -(47) CometScan [native_comet] parquet spark_catalog.default.web_returns +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] Batched: true Location: InMemoryFileIndex [] @@ -308,7 +308,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -123,7 +123,7 @@ Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometC Input [3]: [item_sk#9, d_date#6, cume_sales#11] Arguments: [item_sk#9, d_date#6, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] -(19) CometScan [native_comet] parquet spark_catalog.default.store_sales +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -227,10 +227,10 @@ BroadcastExchange (46) +- * CometColumnarToRow (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) -(42) CometScan [native_comet] parquet spark_catalog.default.date_dim +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt index bccebf349e..30c94463ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt @@ -27,7 +27,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) @@ -35,11 +35,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] CometColumnarExchange [item_sk,d_date] #7 WholeStageCodegen (4) @@ -57,6 +57,6 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt index c6e2fd4218..a05651a422 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt index 74381cf6d8..18bcdb314f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt index 8c17f9f8c6..2fc38bf838 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -45,7 +45,7 @@ Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcha Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt index 6b78c94e2f..b8716d034f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt @@ -20,10 +20,10 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manufact_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -31,11 +31,11 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [d_date_sk,d_qoy] #5 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [s_store_sk] #6 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt index 3d29f3865a..26ce2a5731 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt @@ -27,39 +27,39 @@ : : : : : : :- CometUnion (7) : : : : : : : :- CometProject (3) : : : : : : : : +- CometFilter (2) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : +- CometProject (6) : : : : : : : +- CometFilter (5) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (4) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (4) : : : : : : +- CometBroadcastExchange (11) : : : : : : +- CometProject (10) : : : : : : +- CometFilter (9) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) : : : : : +- CometBroadcastExchange (17) : : : : : +- CometProject (16) : : : : : +- CometFilter (15) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : : : : +- CometBroadcastExchange (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (30) : : : +- CometFilter (29) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) : : +- CometBroadcastExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) : +- CometBroadcastExchange (42) : +- CometProject (41) : +- CometFilter (40) - : +- CometScan [native_comet] parquet spark_catalog.default.store (39) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) +- CometBroadcastExchange (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (45) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -75,7 +75,7 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Arguments: [sold_date_sk#5, customer_sk#6, item_sk#7], [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] -(4) CometScan [native_comet] parquet spark_catalog.default.web_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -95,7 +95,7 @@ Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] -(8) CometScan [native_comet] parquet spark_catalog.default.item +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_class#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -123,7 +123,7 @@ Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] Arguments: [customer_sk#6], [customer_sk#6] -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#20, c_current_addr_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -189,7 +189,7 @@ Input [2]: [c_customer_sk#20, c_current_addr_sk#21] Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] Functions: [] -(28) CometScan [native_comet] parquet spark_catalog.default.store_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] @@ -214,7 +214,7 @@ Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -(33) CometScan [native_comet] parquet spark_catalog.default.customer_address +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -242,7 +242,7 @@ Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -(39) CometScan [native_comet] parquet spark_catalog.default.store +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_county#30, s_state#31] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -270,7 +270,7 @@ Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildR Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -(45) CometScan [native_comet] parquet spark_catalog.default.date_dim +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -340,10 +340,10 @@ BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometProject (61) +- CometFilter (60) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (59) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) -(59) CometScan [native_comet] parquet spark_catalog.default.date_dim +(59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -372,10 +372,10 @@ BroadcastExchange (68) +- * CometColumnarToRow (67) +- CometProject (66) +- CometFilter (65) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (64) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) -(64) CometScan [native_comet] parquet spark_catalog.default.date_dim +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -404,10 +404,10 @@ Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (69) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_comet] parquet spark_catalog.default.date_dim +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#44, d_year#45, d_moy#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -446,10 +446,10 @@ Subquery:5 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (79) +- CometProject (78) +- CometFilter (77) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (76) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) -(76) CometScan [native_comet] parquet spark_catalog.default.date_dim +(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#48, d_year#49, d_moy#50] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt index 29a7c6c022..ea21e4f051 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometUnion [sold_date_sk,customer_sk,item_sk] CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,25 +36,25 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [i_item_sk] #5 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #9 WholeStageCodegen (1) @@ -71,7 +71,7 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #4 WholeStageCodegen (1) CometColumnarToRow @@ -81,19 +81,19 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] CometBroadcastExchange [s_county,s_state] #13 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [(d_month_seq + 1)] #3 ReusedSubquery [(d_month_seq + 3)] #4 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt index 4a8c312ba7..47e0117481 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt index 5634c77a32..dde0a8ebc7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt index ff1e432989..5562e86b91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt @@ -15,24 +15,24 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -43,7 +43,7 @@ : : :- CometProject (33) : : : +- CometBroadcastHashJoin (32) : : : :- CometFilter (30) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (29) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (31) : : +- ReusedExchange (34) : +- ReusedExchange (37) @@ -57,13 +57,13 @@ : :- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometFilter (44) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (43) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) : : +- ReusedExchange (45) : +- ReusedExchange (48) +- ReusedExchange (51) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_color#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -196,7 +196,7 @@ Input [2]: [i_item_id#16, sum#17] Keys [1]: [i_item_id#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(29) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -258,7 +258,7 @@ Input [2]: [i_item_id#26, sum#27] Keys [1]: [i_item_id#26] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -(43) CometScan [native_comet] parquet spark_catalog.default.web_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (63) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_comet] parquet spark_catalog.default.date_dim +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt index 682980d345..88b9477d1d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_id,i_color] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] @@ -52,7 +52,7 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt index bee4d6ae52..eb21a3abd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache Input [3]: [i_item_sk#1, i_brand#2, i_category#3] Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_comet] parquet spark_catalog.default.call_center +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt index 027e80ca6b..b131b85ea9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt @@ -28,23 +28,23 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt index 70f65cbfb1..0307d0df0e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt @@ -14,16 +14,16 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- CometBroadcastExchange (33) : +- CometFilter (32) @@ -35,7 +35,7 @@ : :- CometProject (25) : : +- CometBroadcastHashJoin (24) : : :- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (21) : : +- ReusedExchange (23) : +- ReusedExchange (26) +- CometBroadcastExchange (48) @@ -48,12 +48,12 @@ :- CometProject (40) : +- CometBroadcastHashJoin (39) : :- CometFilter (37) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (36) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (36) : +- ReusedExchange (38) +- ReusedExchange (41) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -65,7 +65,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -93,7 +93,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -147,7 +147,7 @@ Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] Input [2]: [item_id#12, ss_item_rev#13] Condition : isnotnull(ss_item_rev#13) -(21) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -214,7 +214,7 @@ Arguments: [item_id#12], [item_id#22], Inner, ((((cast(ss_item_rev#13 as decimal Input [4]: [item_id#12, ss_item_rev#13, item_id#22, cs_item_rev#23] Arguments: [item_id#12, ss_item_rev#13, cs_item_rev#23], [item_id#12, ss_item_rev#13, cs_item_rev#23] -(36) CometScan [native_comet] parquet spark_catalog.default.web_sales +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] @@ -296,14 +296,14 @@ BroadcastExchange (62) +- CometProject (60) +- CometBroadcastHashJoin (59) :- CometFilter (54) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (53) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) +- CometBroadcastExchange (58) +- CometProject (57) +- CometFilter (56) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) -(53) CometScan [native_comet] parquet spark_catalog.default.date_dim +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -314,7 +314,7 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#10, d_week_seq#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -353,10 +353,10 @@ Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquer * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (63) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_comet] parquet spark_catalog.default.date_dim +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#41, d_week_seq#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt index 5caa1367ee..1cc3e6bfff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (1) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -24,7 +24,7 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] CometFilter [d_date,d_week_seq] @@ -34,17 +34,17 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cs_item_rev] #6 CometFilter [item_id,cs_item_rev] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #4 ReusedExchange [d_date_sk] #5 @@ -70,7 +70,7 @@ WholeStageCodegen (1) CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #4 ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt index d8cc3e369e..4366c901e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt @@ -13,19 +13,19 @@ : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) +- CometBroadcastExchange (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -36,14 +36,14 @@ : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_comet] parquet spark_catalog.default.store (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -55,7 +55,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -97,7 +97,7 @@ Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, s Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(12) CometScan [native_comet] parquet spark_catalog.default.store +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -125,7 +125,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] -(18) CometScan [native_comet] parquet spark_catalog.default.date_dim +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -161,7 +161,7 @@ Input [9]: [d_week_seq#38, ss_store_sk#39, sum#40, sum#41, sum#42, sum#43, sum#4 Keys [2]: [d_week_seq#38, ss_store_sk#39] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#47 END))] -(26) CometScan [native_comet] parquet spark_catalog.default.store +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#48, s_store_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -189,7 +189,7 @@ Arguments: [ss_store_sk#39], [s_store_sk#48], Inner, BuildRight Input [11]: [d_week_seq#38, ss_store_sk#39, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_sk#48, s_store_id#50] Arguments: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50], [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50] -(32) CometScan [native_comet] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#58, d_week_seq#59] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt index 4f08c8c75d..d6ad4a9650 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] @@ -37,8 +37,8 @@ WholeStageCodegen (1) CometBroadcastExchange [s_store_sk,s_store_id] #6 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [d_week_seq] #7 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt index 0c9b427b33..da1b420416 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt @@ -15,22 +15,22 @@ : : : +- CometBroadcastHashJoin (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) : : : +- CometBroadcastExchange (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) : : +- CometBroadcastExchange (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) +- CometBroadcastExchange (32) +- CometProject (31) +- CometBroadcastHashJoin (30) :- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.item (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) +- CometBroadcastExchange (29) +- CometFilter (28) +- CometHashAggregate (27) @@ -38,10 +38,10 @@ +- CometHashAggregate (25) +- CometProject (24) +- CometFilter (23) - +- CometScan [native_comet] parquet spark_catalog.default.item (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) -(1) CometScan [native_comet] parquet spark_catalog.default.customer_address +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -56,7 +56,7 @@ Condition : isnotnull(ca_address_sk#1) Input [2]: [ca_address_sk#1, ca_state#2] Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) CometScan [native_comet] parquet spark_catalog.default.customer +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -80,7 +80,7 @@ Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(9) CometScan [native_comet] parquet spark_catalog.default.store_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -105,7 +105,7 @@ Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -133,7 +133,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(20) CometScan [native_comet] parquet spark_catalog.default.item +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_comet] parquet spark_catalog.default.item +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_current_price#17, i_category#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_comet] parquet spark_catalog.default.date_dim +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt index 135f7fd10f..35d3166cef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt @@ -16,13 +16,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_category,sum,count,i_current_price] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt index 6871295754..e5e165b5ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt @@ -15,24 +15,24 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -43,7 +43,7 @@ : : :- CometProject (33) : : : +- CometBroadcastHashJoin (32) : : : :- CometFilter (30) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (29) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (31) : : +- ReusedExchange (34) : +- ReusedExchange (37) @@ -57,13 +57,13 @@ : :- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometFilter (44) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (43) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) : : +- ReusedExchange (45) : +- ReusedExchange (48) +- ReusedExchange (51) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_category#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -196,7 +196,7 @@ Input [2]: [i_item_id#16, sum#17] Keys [1]: [i_item_id#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(29) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -258,7 +258,7 @@ Input [2]: [i_item_id#26, sum#27] Keys [1]: [i_item_id#26] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -(43) CometScan [native_comet] parquet spark_catalog.default.web_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (63) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_comet] parquet spark_catalog.default.date_dim +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt index 8ff5e75747..914f53e34b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_id,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] @@ -52,7 +52,7 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt index 16ecc9f3dd..4f090653c8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt @@ -18,30 +18,30 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.promotion (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.item (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) +- BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometHashAggregate (61) @@ -58,7 +58,7 @@ : : : :- CometProject (46) : : : : +- CometBroadcastHashJoin (45) : : : : :- CometFilter (43) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (42) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (42) : : : : +- ReusedExchange (44) : : : +- ReusedExchange (47) : : +- ReusedExchange (50) @@ -66,7 +66,7 @@ +- ReusedExchange (56) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -78,7 +78,7 @@ ReadSchema: struct Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(5) CometScan [native_comet] parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -123,7 +123,7 @@ Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -(18) CometScan [native_comet] parquet spark_catalog.default.item +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -151,7 +151,7 @@ Arguments: [ss_item_sk#3], [i_item_sk#12], Inner, BuildRight Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11, i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] Arguments: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -(24) CometScan [native_comet] parquet spark_catalog.default.store_sales +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt index 747b2dbf52..2e8a46fe81 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] CometHashAggregate [ss_store_sk,ss_item_sk,revenue,sum,sum(UnscaledValue(ss_sales_price))] @@ -18,7 +18,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -26,15 +26,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #5 CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometBroadcastExchange [ss_store_sk,ave] #6 CometFilter [ss_store_sk,ave] CometHashAggregate [ss_store_sk,ave,sum,count,avg(revenue)] @@ -46,6 +46,6 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt index 645515cf6d..7599e9ecd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt @@ -17,22 +17,22 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.ship_mode (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) +- CometHashAggregate (45) +- CometExchange (44) +- CometHashAggregate (43) @@ -45,14 +45,14 @@ : : :- CometProject (33) : : : +- CometBroadcastHashJoin (32) : : : :- CometFilter (30) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (29) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (31) : : +- ReusedExchange (34) : +- ReusedExchange (37) +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -105,7 +105,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ Right keys [1]: [ws_bill_customer_sk#10] Join type: LeftAnti Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -173,7 +173,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(28) CometScan [native_comet] parquet spark_catalog.default.customer_address +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -205,7 +205,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#2] Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#18] -(35) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -269,10 +269,10 @@ BroadcastExchange (51) +- * CometColumnarToRow (50) +- CometProject (49) +- CometFilter (48) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (47) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) -(47) CometScan [native_comet] parquet spark_catalog.default.date_dim +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt index c68549d28f..68bf32c40b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -29,11 +29,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -41,7 +41,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -51,7 +51,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -61,7 +61,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -69,4 +69,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt index e8b9572c77..0e32283618 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt @@ -13,26 +13,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) +- CometBroadcastExchange (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_comet] parquet spark_catalog.default.promotion (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -44,7 +44,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -93,7 +93,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_comet] parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -107,7 +107,7 @@ Condition : isnotnull(s_store_sk#7) (12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#7, s_county#8, s_state#9] -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -119,7 +119,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -279,10 +279,10 @@ BroadcastExchange (54) +- * CometColumnarToRow (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (50) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_comet] parquet spark_catalog.default.date_dim +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt index e938e05f2c..9d6ed26262 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt @@ -23,7 +23,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -31,11 +31,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -44,7 +44,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #6 WholeStageCodegen (2) @@ -65,10 +65,10 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_net_profit,ss_sold_date_sk,s_state] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt index 3e03a29a60..9ecf9be21f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt @@ -12,33 +12,33 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) : : :- CometFilter (6) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (5) : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (13) : : +- ReusedExchange (15) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (19) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (20) +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.time_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -57,7 +57,7 @@ Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, s Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(5) CometScan [native_comet] parquet spark_catalog.default.web_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -69,7 +69,7 @@ ReadSchema: struct date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_comet] parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_comet] parquet spark_catalog.default.date_dim +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt index a89be99fc8..c770348a8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt index 538f78b8b3..2de3317bc8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt @@ -15,26 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct= 1) AND (cnt#17 <= 5)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt index 3c2f1f34f6..5d798743dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt index a8677af312..649b5dc4d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,10 +51,10 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) : +- ReusedExchange (44) +- CometBroadcastExchange (68) +- CometHashAggregate (67) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -88,7 +88,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -170,7 +170,7 @@ Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -257,7 +257,7 @@ Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] -(39) CometScan [native_comet] parquet spark_catalog.default.web_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -340,7 +340,7 @@ Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] -(57) CometScan [native_comet] parquet spark_catalog.default.web_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt index d4c76e024d..400468d3e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt @@ -17,20 +17,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,20 +41,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] @@ -66,10 +66,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -82,9 +82,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt index dce46d3120..7afb1e3be6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt @@ -22,19 +22,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -44,14 +44,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -61,14 +61,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -88,11 +88,11 @@ : : :- CometProject (71) : : : +- CometBroadcastHashJoin (70) : : : :- CometFilter (68) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -104,7 +104,7 @@ : : :- CometProject (87) : : : +- CometBroadcastHashJoin (86) : : : :- CometFilter (84) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (83) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) : : : +- ReusedExchange (85) : : +- ReusedExchange (88) : +- CometSort (94) @@ -118,14 +118,14 @@ : :- CometProject (101) : : +- CometBroadcastHashJoin (100) : : :- CometFilter (98) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) : : +- ReusedExchange (99) : +- ReusedExchange (102) +- CometSort (108) +- ReusedExchange (107) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -85,7 +85,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometScan [native_comet] parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -113,7 +113,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] -(15) CometScan [native_comet] parquet spark_catalog.default.customer_address +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [1]: [ca_zip#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -127,7 +127,7 @@ Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVa Input [1]: [ca_zip#12] Arguments: [ca_zip#13], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) AS ca_zip#13] -(18) CometScan [native_comet] parquet spark_catalog.default.customer_address +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#14, ca_zip#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -142,7 +142,7 @@ Condition : isnotnull(ca_address_sk#14) Input [2]: [ca_address_sk#14, ca_zip#15] Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#15, 10, true, false, true) AS ca_zip#16] -(21) CometScan [native_comet] parquet spark_catalog.default.customer +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,10 +256,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (45) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_comet] parquet spark_catalog.default.date_dim +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt index 2e68b4656c..50673ff9c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometExchange [ca_zip] #6 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] CometBroadcastExchange [ca_zip] #7 CometProject [ca_zip] CometFilter [ca_zip,cnt] @@ -47,8 +47,8 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [c_current_addr_sk] #9 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt index 8010bde9b0..bbc01dd8aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt @@ -23,28 +23,28 @@ TakeOrderedAndProject (106) : : : : : :- CometSort (4) : : : : : : +- CometExchange (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : +- CometSort (9) : : : : : +- CometExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) : : : : +- CometBroadcastExchange (15) : : : : +- CometProject (14) : : : : +- CometFilter (13) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (12) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) : : : +- CometBroadcastExchange (21) : : : +- CometProject (20) : : : +- CometFilter (19) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) : : +- CometBroadcastExchange (27) : : +- CometProject (26) : : +- CometFilter (25) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (24) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) : +- CometBroadcastExchange (33) : +- CometProject (32) : +- CometFilter (31) - : +- CometScan [native_comet] parquet spark_catalog.default.promotion (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) :- * HashAggregate (69) : +- * CometColumnarToRow (68) : +- CometExchange (67) @@ -62,17 +62,17 @@ TakeOrderedAndProject (106) : : : : : :- CometSort (43) : : : : : : +- CometExchange (42) : : : : : : +- CometFilter (41) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (40) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) : : : : : +- CometSort (48) : : : : : +- CometExchange (47) : : : : : +- CometProject (46) : : : : : +- CometFilter (45) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (44) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) : : : : +- ReusedExchange (51) : : : +- CometBroadcastExchange (57) : : : +- CometProject (56) : : : +- CometFilter (55) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (54) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) : : +- ReusedExchange (60) : +- ReusedExchange (63) +- * HashAggregate (99) @@ -92,22 +92,22 @@ TakeOrderedAndProject (106) : : : : :- CometSort (73) : : : : : +- CometExchange (72) : : : : : +- CometFilter (71) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (70) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) : : : : +- CometSort (78) : : : : +- CometExchange (77) : : : : +- CometProject (76) : : : : +- CometFilter (75) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (74) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) : : : +- ReusedExchange (81) : : +- CometBroadcastExchange (87) : : +- CometProject (86) : : +- CometFilter (85) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_site (84) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) : +- ReusedExchange (90) +- ReusedExchange (93) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -127,7 +127,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#1 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -(12) CometScan [native_comet] parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -187,7 +187,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -(18) CometScan [native_comet] parquet spark_catalog.default.store +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#16, s_store_id#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -215,7 +215,7 @@ Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(24) CometScan [native_comet] parquet spark_catalog.default.item +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_current_price#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -243,7 +243,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(30) CometScan [native_comet] parquet spark_catalog.default.promotion +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [2]: [p_promo_sk#21, p_channel_tv#22] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -290,7 +290,7 @@ Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_r Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#33, store channel AS channel#34, concat(store, s_store_id#18) AS id#35] -(40) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] @@ -310,7 +310,7 @@ Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIR Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] -(44) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -354,7 +354,7 @@ Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -(54) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -425,7 +425,7 @@ Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_ Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#64, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#65, catalog channel AS channel#66, concat(catalog_page, cp_catalog_page_id#52) AS id#67] -(70) CometScan [native_comet] parquet spark_catalog.default.web_sales +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -445,7 +445,7 @@ Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIR Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] -(74) CometScan [native_comet] parquet spark_catalog.default.web_returns +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -489,7 +489,7 @@ Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -(84) CometScan [native_comet] parquet spark_catalog.default.web_site +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#82, web_site_id#83] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] @@ -598,10 +598,10 @@ BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometProject (109) +- CometFilter (108) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (107) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(107) CometScan [native_comet] parquet spark_catalog.default.date_dim +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt index 9bed5e4a3c..d403cf479b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometExchange [ss_item_sk,ss_ticket_number] #3 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,28 +36,28 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] WholeStageCodegen (2) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow @@ -77,18 +77,18 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometExchange [cs_item_sk,cs_order_number] #11 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan [native_comet] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 WholeStageCodegen (3) @@ -110,17 +110,17 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometExchange [ws_item_sk,ws_order_number] #15 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt index 8e66882ba4..35974f7924 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt @@ -16,15 +16,15 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : : +- CometBroadcastExchange (34) : : +- CometFilter (33) : : +- CometHashAggregate (32) @@ -38,20 +38,20 @@ : : :- CometProject (23) : : : +- CometBroadcastHashJoin (22) : : : :- CometFilter (20) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (19) : : : +- ReusedExchange (21) : : +- ReusedExchange (24) : +- CometBroadcastExchange (40) : +- CometProject (39) : +- CometFilter (38) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (37) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) +- CometBroadcastExchange (46) +- CometProject (45) +- CometFilter (44) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (43) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -63,7 +63,7 @@ ReadSchema: struct= 62.00)) A Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan [native_comet] parquet spark_catalog.default.inventory +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +71,7 @@ Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan [native_comet] parquet spark_catalog.default.store_sales +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#13, ss_sold_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt index 40fc82d84d..28d160a4a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt @@ -14,11 +14,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -26,11 +26,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [ss_item_sk] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt index 840810db81..515e2540f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt @@ -13,16 +13,16 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- CometBroadcastExchange (31) : +- CometHashAggregate (30) @@ -33,7 +33,7 @@ : :- CometProject (24) : : +- CometBroadcastHashJoin (23) : : :- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (20) : : +- ReusedExchange (22) : +- ReusedExchange (25) +- CometBroadcastExchange (45) @@ -45,12 +45,12 @@ :- CometProject (38) : +- CometBroadcastHashJoin (37) : :- CometFilter (35) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (34) : +- ReusedExchange (36) +- ReusedExchange (39) -(1) CometScan [native_comet] parquet spark_catalog.default.store_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -62,7 +62,7 @@ ReadSchema: struct Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -90,7 +90,7 @@ Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -140,7 +140,7 @@ Input [2]: [i_item_id#7, sum#11] Keys [1]: [i_item_id#7] Functions [1]: [sum(sr_return_quantity#2)] -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -203,7 +203,7 @@ Arguments: [item_id#22], [item_id#20], Inner, BuildRight Input [4]: [item_id#22, sr_item_qty#23, item_id#20, cr_item_qty#21] Arguments: [item_id#22, sr_item_qty#23, cr_item_qty#21], [item_id#22, sr_item_qty#23, cr_item_qty#21] -(34) CometScan [native_comet] parquet spark_catalog.default.web_returns +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] Batched: true Location: InMemoryFileIndex [] @@ -281,18 +281,18 @@ BroadcastExchange (63) +- CometProject (61) +- CometBroadcastHashJoin (60) :- CometFilter (51) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) +- CometBroadcastExchange (59) +- CometProject (58) +- CometBroadcastHashJoin (57) - :- CometScan [native_comet] parquet spark_catalog.default.date_dim (52) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) +- CometBroadcastExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (53) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) -(50) CometScan [native_comet] parquet spark_catalog.default.date_dim +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -303,13 +303,13 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(52) CometScan [native_comet] parquet spark_catalog.default.date_dim +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#10, d_week_seq#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(53) CometScan [native_comet] parquet spark_catalog.default.date_dim +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt index bdbaa1502f..9c3886920e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -23,24 +23,24 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cr_item_qty] #7 CometHashAggregate [item_id,cr_item_qty,i_item_id,sum,sum(cr_return_quantity)] @@ -51,7 +51,7 @@ WholeStageCodegen (1) CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #5 ReusedExchange [d_date_sk] #6 @@ -64,7 +64,7 @@ WholeStageCodegen (1) CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #5 ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt index c907048b21..45c375a787 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/explain.txt @@ -14,27 +14,27 @@ TakeOrderedAndProject (33) : : : : +- CometBroadcastHashJoin (8) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) : : : +- CometBroadcastExchange (12) : : : +- CometFilter (11) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.income_band (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.store_returns (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -49,7 +49,7 @@ Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2) Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#1, 16, true, false, true) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#5, 20, true, false, true) AS c_first_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#6, 30, true, false, true) AS c_last_name#9] -(4) CometScan [native_comet] parquet spark_catalog.default.customer_address +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#10, ca_city#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -77,7 +77,7 @@ Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(10) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -101,7 +101,7 @@ Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(15) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -125,7 +125,7 @@ Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(20) CometScan [native_comet] parquet spark_catalog.default.income_band +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -157,7 +157,7 @@ Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_c Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(27) CometScan [native_comet] parquet spark_catalog.default.store_returns +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt index 6936907ec8..cc7cf901e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84/simplified.txt @@ -15,21 +15,21 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometScan [native_comet] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt index 4bab741e14..c263f03d76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt @@ -20,36 +20,36 @@ : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometBroadcastExchange (3) : : : : : : : +- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_page (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometProject (16) : : : : +- CometFilter (15) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) +- CometBroadcastExchange (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_comet] parquet spark_catalog.default.reason (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -65,7 +65,7 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometScan [native_comet] parquet spark_catalog.default.web_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -89,7 +89,7 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(9) CometScan [native_comet] parquet spark_catalog.default.web_page +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -113,7 +113,7 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(14) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -141,7 +141,7 @@ Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_s Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -169,7 +169,7 @@ Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status# Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(26) CometScan [native_comet] parquet spark_catalog.default.customer_address +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -197,7 +197,7 @@ Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(32) CometScan [native_comet] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -225,7 +225,7 @@ Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(38) CometScan [native_comet] parquet spark_catalog.default.reason +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#35, r_reason_desc#36] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -281,10 +281,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt index df11eb5798..52cf7b57a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -29,30 +29,30 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometBroadcastExchange [wp_web_page_sk] #4 CometFilter [wp_web_page_sk] - CometScan [native_comet] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #7 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt index c0054aaa80..1af5e2612c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt @@ -14,18 +14,18 @@ TakeOrderedAndProject (24) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -65,7 +65,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -141,10 +141,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_comet] parquet spark_catalog.default.date_dim +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt index f595259476..92a7eb7e8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt @@ -17,7 +17,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -25,12 +25,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt index f2454e937d..fc6b12b834 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt @@ -15,15 +15,15 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : +- BroadcastExchange (31) : +- * CometColumnarToRow (30) : +- CometHashAggregate (29) @@ -34,7 +34,7 @@ : :- CometProject (23) : : +- CometBroadcastHashJoin (22) : : :- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (19) : : +- ReusedExchange (21) : +- ReusedExchange (24) +- BroadcastExchange (45) @@ -47,12 +47,12 @@ :- CometProject (37) : +- CometBroadcastHashJoin (36) : :- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : +- ReusedExchange (35) +- ReusedExchange (38) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -92,7 +92,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_comet] parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -137,7 +137,7 @@ Functions: [] (18) CometColumnarToRow [codegen id : 3] Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -(19) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -200,7 +200,7 @@ Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_ Join type: LeftAnti Join condition: None -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -295,10 +295,10 @@ BroadcastExchange (56) +- * CometColumnarToRow (55) +- CometProject (54) +- CometFilter (53) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (52) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) -(52) CometScan [native_comet] parquet spark_catalog.default.date_dim +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt index 6b09d49e07..f687139735 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt @@ -18,7 +18,7 @@ WholeStageCodegen (4) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -26,15 +26,15 @@ WholeStageCodegen (4) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (1) @@ -48,7 +48,7 @@ WholeStageCodegen (4) CometProject [cs_bill_customer_sk,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 @@ -65,7 +65,7 @@ WholeStageCodegen (4) CometProject [ws_bill_customer_sk,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt index 6ff7198461..2e49d20af4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt @@ -18,19 +18,19 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) +: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (16) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) : : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) @@ -44,12 +44,12 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (26) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (32) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) : : : : : +- * CometColumnarToRow (65) @@ -64,12 +64,12 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (47) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (53) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) : : : : +- * CometColumnarToRow (86) @@ -84,12 +84,12 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (68) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (74) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) : : : +- * CometColumnarToRow (107) @@ -104,12 +104,12 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (89) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (95) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) : : +- * CometColumnarToRow (128) @@ -124,12 +124,12 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (110) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (116) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) : +- * CometColumnarToRow (149) @@ -144,12 +144,12 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (131) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (137) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) +- * CometColumnarToRow (170) @@ -164,16 +164,16 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (152) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (158) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -188,7 +188,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -216,7 +216,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -244,7 +244,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -288,7 +288,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometScan [native_comet] parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -315,7 +315,7 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometScan [native_comet] parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -379,7 +379,7 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometScan [native_comet] parquet spark_catalog.default.store_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -406,7 +406,7 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometScan [native_comet] parquet spark_catalog.default.time_dim +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -470,7 +470,7 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometScan [native_comet] parquet spark_catalog.default.store_sales +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -497,7 +497,7 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometScan [native_comet] parquet spark_catalog.default.time_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -561,7 +561,7 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometScan [native_comet] parquet spark_catalog.default.store_sales +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -588,7 +588,7 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometScan [native_comet] parquet spark_catalog.default.time_dim +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -652,7 +652,7 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometScan [native_comet] parquet spark_catalog.default.store_sales +(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -679,7 +679,7 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometScan [native_comet] parquet spark_catalog.default.time_dim +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -743,7 +743,7 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometScan [native_comet] parquet spark_catalog.default.store_sales +(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -770,7 +770,7 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometScan [native_comet] parquet spark_catalog.default.time_dim +(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -834,7 +834,7 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometScan [native_comet] parquet spark_catalog.default.store_sales +(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -861,7 +861,7 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometScan [native_comet] parquet spark_catalog.default.time_dim +(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt index c47266d366..3490714189 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt @@ -19,19 +19,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -48,12 +48,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 @@ -71,12 +71,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 @@ -94,12 +94,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 @@ -117,12 +117,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 @@ -140,12 +140,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 @@ -163,12 +163,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt index 15f4eace86..47ce0d4a91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -45,7 +45,7 @@ Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarchar Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) AS i_class#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) AS i_category#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt index f7f05c899a..140fe297a1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt @@ -20,10 +20,10 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] CometFilter [i_item_sk,i_brand,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -31,11 +31,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt index 2f50da0bab..b3f32555de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt @@ -2,10 +2,10 @@ * Project (4) +- * CometColumnarToRow (3) +- CometFilter (2) - +- CometScan [native_comet] parquet spark_catalog.default.reason (1) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) -(1) CometScan [native_comet] parquet spark_catalog.default.reason +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -33,10 +33,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) -(5) CometScan [native_comet] parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -84,10 +84,10 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (13) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -135,10 +135,10 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -186,10 +186,10 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (29) -(29) CometScan [native_comet] parquet spark_catalog.default.store_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -237,10 +237,10 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (37) -(37) CometScan [native_comet] parquet spark_catalog.default.store_sales +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt index 8644223a0b..510cf60bd8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 @@ -23,7 +23,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 @@ -49,7 +49,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 @@ -62,10 +62,10 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 CometColumnarToRow InputAdapter CometFilter [r_reason_sk] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt index b3916111e7..08ae744b5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt @@ -13,19 +13,19 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometScan [native_comet] parquet spark_catalog.default.web_page (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometHashAggregate (43) @@ -39,16 +39,16 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -63,7 +63,7 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -91,7 +91,7 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -119,7 +119,7 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.web_page +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -163,7 +163,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometScan [native_comet] parquet spark_catalog.default.web_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -190,7 +190,7 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometScan [native_comet] parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt index 1a5ae5b626..2db3b6c957 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometScan [native_comet] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt index ef109a81aa..0ba113dd4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt @@ -19,32 +19,32 @@ : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometProject (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometFilter (5) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) : : +- CometBroadcastExchange (23) : : +- CometProject (22) : : +- CometFilter (21) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) -(1) CometScan [native_comet] parquet spark_catalog.default.call_center +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -59,7 +59,7 @@ Condition : isnotnull(cc_call_center_sk#1) Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cc_call_center_id#2, 16, true, false, true) AS cc_call_center_id#5, cc_name#3, cc_manager#4] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -84,7 +84,7 @@ Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -112,7 +112,7 @@ Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(15) CometScan [native_comet] parquet spark_catalog.default.customer +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -136,7 +136,7 @@ Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_address +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -164,7 +164,7 @@ Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -(26) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -192,7 +192,7 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -(32) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#25, hd_buy_potential#26] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -252,10 +252,10 @@ BroadcastExchange (48) +- * CometColumnarToRow (47) +- CometProject (46) +- CometFilter (45) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (44) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) -(44) CometScan [native_comet] parquet spark_catalog.default.date_dim +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt index 994ca92319..5be6489e2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt @@ -20,10 +20,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -31,23 +31,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #7 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk] #9 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt index c12bb60534..cdc8dbc3bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,15 +23,15 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -71,7 +71,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -(9) CometScan [native_comet] parquet spark_catalog.default.web_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Condition : isnotnull(ws_item_sk#7) -(11) CometScan [native_comet] parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_comet] parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt index 6be4fb9e4f..7a1825cc9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt @@ -11,7 +11,7 @@ WholeStageCodegen (1) CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,sum,count,avg(UnscaledValue(ws_ext_discount_amt))] @@ -32,10 +32,10 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt index 498abd2426..799d860c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt @@ -11,19 +11,19 @@ : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometScan [native_comet] parquet spark_catalog.default.reason (12) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -73,7 +73,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometScan [native_comet] parquet spark_catalog.default.reason +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt index 9b21bd273a..d761953c97 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/simplified.txt @@ -12,13 +12,13 @@ WholeStageCodegen (1) CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometExchange [ss_item_sk,ss_ticket_number] #2 CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometExchange [sr_item_sk,sr_ticket_number] #3 CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [r_reason_sk] #4 CometProject [r_reason_sk] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt index 002c60f908..3caf1593da 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (12) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (23) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -92,7 +92,7 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometScan [native_comet] parquet spark_catalog.default.web_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -115,7 +115,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometScan [native_comet] parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometScan [native_comet] parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometScan [native_comet] parquet spark_catalog.default.web_site +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt index 956fe14b3e..068a535b6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [wr_order_number] CometExchange [wr_order_number] #4 CometProject [wr_order_number] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt index 220fbfb3a8..5ecdefc563 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt @@ -18,14 +18,14 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometProject (14) : : : : +- CometSortMergeJoin (13) : : : : :- CometSort (10) : : : : : +- CometExchange (9) : : : : : +- CometProject (8) : : : : : +- CometFilter (7) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) : : : : +- CometSort (12) : : : : +- ReusedExchange (11) : : : +- CometProject (28) @@ -34,7 +34,7 @@ : : : : +- CometExchange (19) : : : : +- CometProject (18) : : : : +- CometFilter (17) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (16) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) : : : +- CometProject (26) : : : +- CometSortMergeJoin (25) : : : :- CometSort (22) @@ -44,18 +44,18 @@ : : +- CometBroadcastExchange (33) : : +- CometProject (32) : : +- CometFilter (31) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : +- CometBroadcastExchange (39) : +- CometProject (38) : +- CometFilter (37) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (36) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) +- CometBroadcastExchange (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -78,7 +78,7 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -122,7 +122,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometScan [native_comet] parquet spark_catalog.default.web_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -182,7 +182,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -210,7 +210,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_address +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#21, ca_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -238,7 +238,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometScan [native_comet] parquet spark_catalog.default.web_site +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#23, web_company_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt index be2453ac89..6f07040b87 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt @@ -21,14 +21,14 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 CometProject [wr_order_number] @@ -37,7 +37,7 @@ WholeStageCodegen (2) CometExchange [wr_order_number] #4 CometProject [wr_order_number] CometFilter [wr_order_number,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] @@ -47,12 +47,12 @@ WholeStageCodegen (2) CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt index 86b62b1982..f90d8b12b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt @@ -11,22 +11,22 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometScan [native_comet] parquet spark_catalog.default.store (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -69,7 +69,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -97,7 +97,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt index 34e10d3a24..36f4009885 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt index 527d11b75e..420f37fccc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt @@ -11,29 +11,29 @@ : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (2) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) - :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (12) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) +- ReusedExchange (13) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] ReadSchema: struct -(2) CometScan [native_comet] parquet spark_catalog.default.date_dim +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -79,7 +79,7 @@ Functions: [] Input [2]: [customer_sk#7, item_sk#8] Arguments: [customer_sk#7, item_sk#8], [customer_sk#7 ASC NULLS FIRST, item_sk#8 ASC NULLS FIRST] -(12) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -149,10 +149,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt index f2cb24041e..49b28b94be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_customer_sk,ss_item_sk] CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -20,17 +20,17 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] CometExchange [cs_bill_customer_sk,cs_item_sk] #5 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt index 68a1ee0c6d..4cc725ef2b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt @@ -16,18 +16,18 @@ :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -67,7 +67,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -150,10 +150,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt index cb1b3359a9..f563efa55d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (3) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -29,12 +29,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt index cbaeb3a9ae..7e57135c10 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt @@ -13,24 +13,24 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.ship_mode (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) : +- CometBroadcastExchange (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) +- CometBroadcastExchange (22) +- CometProject (21) +- CometFilter (20) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -41,7 +41,7 @@ ReadSchema: struct (avg(ctr_total_ret Output [2]: [ctr_customer_sk#11, ctr_store_sk#12] Input [5]: [ctr_customer_sk#11, ctr_store_sk#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#29, ctr_store_sk#22] -(33) CometScan [native_comet] parquet spark_catalog.default.store +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#30, s_state#31] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -235,7 +235,7 @@ Join condition: None Output [1]: [ctr_customer_sk#11] Input [3]: [ctr_customer_sk#11, ctr_store_sk#12, s_store_sk#30] -(40) CometScan [native_comet] parquet spark_catalog.default.customer +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#32, c_customer_id#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -278,10 +278,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt index 838ad0bd97..20912acfb5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [c_customer_id] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -26,11 +26,11 @@ TakeOrderedAndProject [c_customer_id] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #4 WholeStageCodegen (4) @@ -52,7 +52,7 @@ TakeOrderedAndProject [c_customer_id] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #3 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [c_customer_id] InputAdapter CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (6) @@ -70,4 +70,4 @@ TakeOrderedAndProject [c_customer_id] InputAdapter CometProject [c_customer_id] [c_customer_sk,c_customer_id] CometFilter [c_customer_sk,c_customer_id] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt index 78c5aa9c39..e253b8ca7b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt @@ -15,40 +15,40 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (13) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) : : : +- ReusedExchange (14) : : +- BroadcastExchange (25) : : +- * CometColumnarToRow (24) : : +- CometProject (23) : : +- CometBroadcastHashJoin (22) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (20) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) : : +- ReusedExchange (21) : +- BroadcastExchange (33) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,14 +59,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_county#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#4] Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt index dcab21d624..e7193f87e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt index cce8a7ee90..860aeed1b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt @@ -18,13 +18,13 @@ TakeOrderedAndProject (80) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- BroadcastExchange (38) : : +- * HashAggregate (37) : : +- * CometColumnarToRow (36) @@ -37,13 +37,13 @@ TakeOrderedAndProject (80) : : : +- CometBroadcastHashJoin (26) : : : :- CometProject (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (25) : : : +- CometFilter (24) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : +- CometBroadcastExchange (30) : : +- CometFilter (29) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (28) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (28) : +- BroadcastExchange (58) : +- * Filter (57) : +- * HashAggregate (56) @@ -57,10 +57,10 @@ TakeOrderedAndProject (80) : : +- CometBroadcastHashJoin (47) : : :- CometProject (43) : : : +- CometFilter (42) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (41) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) : : +- CometBroadcastExchange (46) : : +- CometFilter (45) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (44) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) : +- ReusedExchange (49) +- BroadcastExchange (77) +- * HashAggregate (76) @@ -74,14 +74,14 @@ TakeOrderedAndProject (80) : +- CometBroadcastHashJoin (67) : :- CometProject (63) : : +- CometFilter (62) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (61) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (61) : +- CometBroadcastExchange (66) : +- CometFilter (65) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (64) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (64) +- ReusedExchange (69) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -96,7 +96,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharC Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -121,7 +121,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -173,7 +173,7 @@ Results [2]: [c_customer_id#9 AS customer_id#25, MakeDecimal(sum(UnscaledValue(( Input [2]: [customer_id#25, year_total#26] Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.00)) -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -188,7 +188,7 @@ Condition : (isnotnull(c_customer_sk#27) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Arguments: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)) AS c_customer_id#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#29, 20)) AS c_first_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#30, 30)) AS c_last_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#31, 1)) AS c_preferred_cust_flag#38, c_birth_country#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#33, 13)) AS c_login#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#34, 50)) AS c_email_address#40] -(23) CometScan [native_comet] parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] @@ -213,7 +213,7 @@ Arguments: [c_customer_sk#27], [ss_customer_sk#41], Inner, BuildRight Input [12]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44] Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44] -(28) CometScan [native_comet] parquet spark_catalog.default.date_dim +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#46, d_year#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -275,7 +275,7 @@ Join condition: None Output [4]: [customer_id#25, year_total#26, customer_preferred_cust_flag#51, year_total#52] Input [5]: [customer_id#25, year_total#26, customer_id#50, customer_preferred_cust_flag#51, year_total#52] -(41) CometScan [native_comet] parquet spark_catalog.default.customer +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -290,7 +290,7 @@ Condition : (isnotnull(c_customer_sk#53) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#54, 16)) AS c_customer_id#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#55, 20)) AS c_first_name#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#56, 30)) AS c_last_name#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#57, 1)) AS c_preferred_cust_flag#64, c_birth_country#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#59, 13)) AS c_login#65, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#60, 50)) AS c_email_address#66] -(44) CometScan [native_comet] parquet spark_catalog.default.web_sales +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#67, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] Batched: true Location: InMemoryFileIndex [] @@ -369,7 +369,7 @@ Join condition: None Output [5]: [customer_id#25, year_total#26, customer_preferred_cust_flag#51, year_total#52, year_total#78] Input [6]: [customer_id#25, year_total#26, customer_preferred_cust_flag#51, year_total#52, customer_id#77, year_total#78] -(61) CometScan [native_comet] parquet spark_catalog.default.customer +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -384,7 +384,7 @@ Condition : (isnotnull(c_customer_sk#79) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] Arguments: [c_customer_sk#79, c_customer_id#87, c_first_name#88, c_last_name#89, c_preferred_cust_flag#90, c_birth_country#84, c_login#91, c_email_address#92], [c_customer_sk#79, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#80, 16)) AS c_customer_id#87, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#81, 20)) AS c_first_name#88, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#82, 30)) AS c_last_name#89, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#83, 1)) AS c_preferred_cust_flag#90, c_birth_country#84, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#85, 13)) AS c_login#91, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#86, 50)) AS c_email_address#92] -(64) CometScan [native_comet] parquet spark_catalog.default.web_sales +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#93, ws_ext_discount_amt#94, ws_ext_list_price#95, ws_sold_date_sk#96] Batched: true Location: InMemoryFileIndex [] @@ -469,10 +469,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (84) +- * CometColumnarToRow (83) +- CometFilter (82) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (81) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (81) -(81) CometScan [native_comet] parquet spark_catalog.default.date_dim +(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -494,10 +494,10 @@ Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#44 IN d BroadcastExchange (88) +- * CometColumnarToRow (87) +- CometFilter (86) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (85) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (85) -(85) CometScan [native_comet] parquet spark_catalog.default.date_dim +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#46, d_year#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt index 0301ffa55b..bf5fb2262d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt @@ -21,20 +21,20 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -52,20 +52,20 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) @@ -84,10 +84,10 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 InputAdapter @@ -107,9 +107,9 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt index 54aaac6495..74e9b1b5e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt @@ -15,18 +15,18 @@ TakeOrderedAndProject (25) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -148,10 +148,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt index 42df6637ca..72fe5bab30 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,12 +30,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt index 936cdd09f0..9391720c65 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt @@ -15,28 +15,28 @@ : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) +- CometBroadcastExchange (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -48,7 +48,7 @@ ReadSchema: struct= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometScan [native_comet] parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -72,7 +72,7 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -100,7 +100,7 @@ Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -128,7 +128,7 @@ Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -156,7 +156,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -(26) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#24, hd_dep_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -211,10 +211,10 @@ BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) -(36) CometScan [native_comet] parquet spark_catalog.default.date_dim +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt index ca146ce742..d2f70ac58f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt @@ -18,7 +18,7 @@ WholeStageCodegen (2) CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -26,22 +26,22 @@ WholeStageCodegen (2) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt index 9856a9547c..c694f1e88e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt @@ -19,12 +19,12 @@ TakeOrderedAndProject (108) : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -35,25 +35,25 @@ TakeOrderedAndProject (108) : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -61,18 +61,18 @@ TakeOrderedAndProject (108) : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) :- * Project (84) : +- * Filter (83) : +- * HashAggregate (82) @@ -86,7 +86,7 @@ TakeOrderedAndProject (108) : : +- CometBroadcastHashJoin (73) : : :- CometBroadcastHashJoin (71) : : : :- CometFilter (69) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (68) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (68) : : : +- ReusedExchange (70) : : +- ReusedExchange (72) : +- ReusedExchange (75) @@ -103,13 +103,13 @@ TakeOrderedAndProject (108) : +- CometBroadcastHashJoin (90) : :- CometBroadcastHashJoin (88) : : :- CometFilter (86) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (85) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (85) : : +- ReusedExchange (87) : +- ReusedExchange (89) +- ReusedExchange (92) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -121,7 +121,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -132,7 +132,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -155,7 +155,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -191,7 +191,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -267,7 +267,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -334,7 +334,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -366,7 +366,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -426,7 +426,7 @@ Condition : (isnotnull(sales#54) AND (cast(sales#54 as decimal(32,6)) > cast(Sub Output [6]: [sales#54, number_sales#55, store AS channel#58, i_brand_id#40 AS i_brand_id#59, i_class_id#41 AS i_class_id#60, i_category_id#42 AS i_category_id#61] Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#54, number_sales#55] -(68) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#62, cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65] Batched: true Location: InMemoryFileIndex [] @@ -502,7 +502,7 @@ Condition : (isnotnull(sales#81) AND (cast(sales#81 as decimal(32,6)) > cast(Reu Output [6]: [sales#81, number_sales#82, catalog AS channel#83, i_brand_id#69, i_class_id#70, i_category_id#71] Input [5]: [i_brand_id#69, i_class_id#70, i_category_id#71, sales#81, number_sales#82] -(85) CometScan [native_comet] parquet spark_catalog.default.web_sales +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#84, ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] @@ -620,19 +620,19 @@ Subquery:1 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquer +- CometUnion (121) :- CometProject (112) : +- CometBroadcastHashJoin (111) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (109) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (109) : +- ReusedExchange (110) :- CometProject (116) : +- CometBroadcastHashJoin (115) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (113) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (113) : +- ReusedExchange (114) +- CometProject (120) +- CometBroadcastHashJoin (119) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (117) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (117) +- ReusedExchange (118) -(109) CometScan [native_comet] parquet spark_catalog.default.store_sales +(109) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123] Batched: true Location: InMemoryFileIndex [] @@ -651,7 +651,7 @@ Arguments: [ss_sold_date_sk#123], [d_date_sk#125], Inner, BuildRight Input [4]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123, d_date_sk#125] Arguments: [quantity#126, list_price#127], [ss_quantity#121 AS quantity#126, ss_list_price#122 AS list_price#127] -(113) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130] Batched: true Location: InMemoryFileIndex [] @@ -670,7 +670,7 @@ Arguments: [cs_sold_date_sk#130], [d_date_sk#132], Inner, BuildRight Input [4]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130, d_date_sk#132] Arguments: [quantity#133, list_price#134], [cs_quantity#128 AS quantity#133, cs_list_price#129 AS list_price#134] -(117) CometScan [native_comet] parquet spark_catalog.default.web_sales +(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#135, ws_list_price#136, ws_sold_date_sk#137] Batched: true Location: InMemoryFileIndex [] @@ -729,10 +729,10 @@ BroadcastExchange (131) +- * CometColumnarToRow (130) +- CometProject (129) +- CometFilter (128) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (127) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (127) -(127) CometScan [native_comet] parquet spark_catalog.default.date_dim +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -759,10 +759,10 @@ BroadcastExchange (136) +- * CometColumnarToRow (135) +- CometProject (134) +- CometFilter (133) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (132) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (132) -(132) CometScan [native_comet] parquet spark_catalog.default.date_dim +(132) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#148] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt index 6625cacadc..685b4b4082 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt @@ -25,17 +25,17 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] @@ -52,7 +52,7 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -60,12 +60,12 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -76,7 +76,7 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) @@ -84,26 +84,26 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 CometProject [i_brand_id,i_class_id,i_category_id] @@ -111,19 +111,19 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #4 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] WholeStageCodegen (4) Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] Filter [sales] @@ -142,7 +142,7 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 @@ -165,7 +165,7 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),su CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt index 86eef700e0..1410b0d8ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt @@ -13,12 +13,12 @@ TakeOrderedAndProject (88) : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -29,25 +29,25 @@ TakeOrderedAndProject (88) : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -55,18 +55,18 @@ TakeOrderedAndProject (88) : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- BroadcastExchange (86) +- * Filter (85) +- * HashAggregate (84) @@ -80,16 +80,16 @@ TakeOrderedAndProject (88) : +- CometBroadcastHashJoin (72) : :- CometBroadcastHashJoin (70) : : :- CometFilter (68) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (67) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (67) : : +- ReusedExchange (69) : +- ReusedExchange (71) +- CometBroadcastExchange (77) +- CometProject (76) +- CometFilter (75) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -101,7 +101,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -112,7 +112,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -135,7 +135,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -171,7 +171,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -247,7 +247,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -314,7 +314,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -346,7 +346,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -402,7 +402,7 @@ Results [6]: [store AS channel#55, i_brand_id#40, i_class_id#41, i_category_id#4 Input [6]: [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57] Condition : (isnotnull(sales#56) AND (cast(sales#56 as decimal(32,6)) > cast(Subquery scalar-subquery#58, [id=#59] as decimal(32,6)))) -(67) CometScan [native_comet] parquet spark_catalog.default.store_sales +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] @@ -434,7 +434,7 @@ Arguments: [ss_item_sk#60], [i_item_sk#66], Inner, BuildRight Input [8]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] Arguments: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69], [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69] -(74) CometScan [native_comet] parquet spark_catalog.default.date_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -515,19 +515,19 @@ Subquery:1 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquer +- CometUnion (101) :- CometProject (92) : +- CometBroadcastHashJoin (91) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (89) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) : +- ReusedExchange (90) :- CometProject (96) : +- CometBroadcastHashJoin (95) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (93) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (93) : +- ReusedExchange (94) +- CometProject (100) +- CometBroadcastHashJoin (99) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (97) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) +- ReusedExchange (98) -(89) CometScan [native_comet] parquet spark_catalog.default.store_sales +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] @@ -546,7 +546,7 @@ Arguments: [ss_sold_date_sk#87], [d_date_sk#89], Inner, BuildRight Input [4]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87, d_date_sk#89] Arguments: [quantity#90, list_price#91], [ss_quantity#85 AS quantity#90, ss_list_price#86 AS list_price#91] -(93) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94] Batched: true Location: InMemoryFileIndex [] @@ -565,7 +565,7 @@ Arguments: [cs_sold_date_sk#94], [d_date_sk#96], Inner, BuildRight Input [4]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94, d_date_sk#96] Arguments: [quantity#97, list_price#98], [cs_quantity#92 AS quantity#97, cs_list_price#93 AS list_price#98] -(97) CometScan [native_comet] parquet spark_catalog.default.web_sales +(97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] @@ -624,10 +624,10 @@ BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometProject (109) +- CometFilter (108) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (107) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(107) CometScan [native_comet] parquet spark_catalog.default.date_dim +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -655,10 +655,10 @@ Subquery:7 Hosting operator id = 107 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -681,10 +681,10 @@ BroadcastExchange (120) +- * CometColumnarToRow (119) +- CometProject (118) +- CometFilter (117) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (116) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (116) -(116) CometScan [native_comet] parquet spark_catalog.default.date_dim +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#116] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -721,10 +721,10 @@ BroadcastExchange (125) +- * CometColumnarToRow (124) +- CometProject (123) +- CometFilter (122) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (121) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) -(121) CometScan [native_comet] parquet spark_catalog.default.date_dim +(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -752,10 +752,10 @@ Subquery:16 Hosting operator id = 121 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (129) +- CometProject (128) +- CometFilter (127) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (126) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (126) -(126) CometScan [native_comet] parquet spark_catalog.default.date_dim +(126) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt index 8d14d0d6cf..812bf0d7dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt @@ -15,17 +15,17 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] @@ -42,7 +42,7 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -51,19 +51,19 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -74,7 +74,7 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) @@ -82,26 +82,26 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -109,20 +109,20 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 InputAdapter BroadcastExchange #15 @@ -143,7 +143,7 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #17 WholeStageCodegen (1) @@ -152,19 +152,19 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #6 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt index aa34a97439..831a27f9e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt @@ -12,21 +12,21 @@ TakeOrderedAndProject (25) : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) +- CometBroadcastExchange (17) +- CometProject (16) +- CometFilter (15) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.customer +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#5, c_current_addr_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -62,7 +62,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -90,7 +90,7 @@ Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -153,10 +153,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt index 9e2a625f7c..431c3b5a04 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt @@ -15,7 +15,7 @@ TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -23,15 +23,15 @@ TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt index 8e75bc8080..b9503e8081 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (12) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (23) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.call_center (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -92,7 +92,7 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -115,7 +115,7 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometScan [native_comet] parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometScan [native_comet] parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometScan [native_comet] parquet spark_catalog.default.call_center +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt index 7a086b04dc..74c7d994c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [cs_order_number] #2 CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] CometSort [cs_warehouse_sk,cs_order_number] CometExchange [cs_order_number] #3 CometProject [cs_warehouse_sk,cs_order_number] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] CometSort [cr_order_number] CometExchange [cr_order_number] #4 CometProject [cr_order_number] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cc_call_center_sk] #7 CometProject [cc_call_center_sk] CometFilter [cc_call_center_sk,cc_county] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt index f5afdd680b..73f9928327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt @@ -20,33 +20,33 @@ TakeOrderedAndProject (45) : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_comet] parquet spark_catalog.default.store (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.item (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -58,7 +58,7 @@ ReadSchema: struct Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -66,7 +66,7 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometScan [native_comet] parquet spark_catalog.default.date_dim +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -118,7 +118,7 @@ Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sal Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))#33] Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END))#27,17,2) AS sun_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END))#28,17,2) AS mon_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END))#29,17,2) AS tue_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END))#30,17,2) AS wed_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END))#31,17,2) AS thu_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END))#32,17,2) AS fri_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))#33,17,2) AS sat_sales#40] -(17) CometScan [native_comet] parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#41, d_year#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -163,7 +163,7 @@ Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN sal Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN sales_price#60 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN sales_price#60 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN sales_price#60 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN sales_price#60 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN sales_price#60 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN sales_price#60 END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN sales_price#60 END))#33] Results [8]: [d_week_seq#51, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN sales_price#60 END))#27,17,2) AS sun_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN sales_price#60 END))#28,17,2) AS mon_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN sales_price#60 END))#29,17,2) AS tue_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN sales_price#60 END))#30,17,2) AS wed_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN sales_price#60 END))#31,17,2) AS thu_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN sales_price#60 END))#32,17,2) AS fri_sales#66, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN sales_price#60 END))#33,17,2) AS sat_sales#67] -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#68, d_year#69] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt index bfe344412d..be2dcedb97 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt @@ -20,13 +20,13 @@ WholeStageCodegen (7) CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] CometUnion [sold_date_sk,sales_price] CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) @@ -34,7 +34,7 @@ WholeStageCodegen (7) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (5) @@ -51,4 +51,4 @@ WholeStageCodegen (7) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt index 3171e32c27..14b1aaff57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt @@ -15,18 +15,18 @@ TakeOrderedAndProject (25) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -148,10 +148,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt index 4add1be0db..8eeac91635 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,12 +30,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt index 5e66478bba..ad000b7011 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt @@ -13,20 +13,20 @@ TakeOrderedAndProject (25) : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_comet] parquet spark_catalog.default.item (8) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -38,7 +38,7 @@ ReadSchema: struct -(2) CometScan [native_comet] parquet spark_catalog.default.store_sales +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -89,7 +89,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -117,7 +117,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(10) CometScan [native_comet] parquet spark_catalog.default.item +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -187,7 +187,7 @@ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ (25) CometColumnarToRow [codegen id : 1] Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(26) CometScan [native_comet] parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -202,7 +202,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(29) CometScan [native_comet] parquet spark_catalog.default.customer +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -285,7 +285,7 @@ Join condition: None Output [1]: [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#33] Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#32] -(47) CometScan [native_comet] parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] Batched: true Location: InMemoryFileIndex [] @@ -393,10 +393,10 @@ BroadcastExchange (74) +- * CometColumnarToRow (73) +- CometProject (72) +- CometFilter (71) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (70) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (70) -(70) CometScan [native_comet] parquet spark_catalog.default.date_dim +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#32, d_year#55, d_moy#56] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -423,10 +423,10 @@ BroadcastExchange (79) +- * CometColumnarToRow (78) +- CometProject (77) +- CometFilter (76) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (75) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (75) -(75) CometScan [native_comet] parquet spark_catalog.default.date_dim +(75) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -463,15 +463,15 @@ Subquery:3 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquer :- CometProject (84) : +- CometBroadcastHashJoin (83) : :- CometFilter (81) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (80) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (80) : +- ReusedExchange (82) +- CometBroadcastExchange (88) +- CometProject (87) +- CometFilter (86) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (85) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (85) -(80) CometScan [native_comet] parquet spark_catalog.default.store_sales +(80) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] @@ -495,7 +495,7 @@ Arguments: [ss_customer_sk#57], [c_customer_sk#62], Inner, BuildRight Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62] Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62] -(85) CometScan [native_comet] parquet spark_catalog.default.date_dim +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#63, d_year#64] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -573,10 +573,10 @@ BroadcastExchange (104) +- * CometColumnarToRow (103) +- CometProject (102) +- CometFilter (101) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (100) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (100) -(100) CometScan [native_comet] parquet spark_catalog.default.date_dim +(100) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#63, d_year#64] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt index d5c48d451e..72e974faa4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt @@ -20,7 +20,7 @@ WholeStageCodegen (12) CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -28,7 +28,7 @@ WholeStageCodegen (12) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -40,7 +40,7 @@ WholeStageCodegen (12) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) @@ -48,14 +48,14 @@ WholeStageCodegen (12) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] InputAdapter WholeStageCodegen (3) Sort [c_customer_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (12) CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #13 WholeStageCodegen (1) @@ -90,12 +90,12 @@ WholeStageCodegen (12) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] CometColumnarToRow InputAdapter @@ -108,10 +108,10 @@ WholeStageCodegen (12) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] InputAdapter ReusedExchange [d_date_sk] #3 WholeStageCodegen (10) @@ -127,7 +127,7 @@ WholeStageCodegen (12) CometExchange [ws_bill_customer_sk] #15 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [item_sk] #4 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt index 240d686615..6e27b0f371 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject (93) : : : : +- CometProject (23) : : : : +- CometBroadcastHashJoin (22) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (21) : : : : +- CometProject (20) : : : : +- CometFilter (19) @@ -28,14 +28,14 @@ TakeOrderedAndProject (93) : : : : :- CometProject (10) : : : : : +- CometBroadcastHashJoin (9) : : : : : :- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : : +- CometBroadcastExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) : : : : +- CometBroadcastExchange (13) : : : : +- CometFilter (12) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : +- * Sort (42) : : : +- * Project (41) : : : +- * Filter (40) @@ -48,10 +48,10 @@ TakeOrderedAndProject (93) : : : +- CometBroadcastHashJoin (33) : : : :- CometProject (29) : : : : +- CometFilter (28) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (27) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (27) : : : +- CometBroadcastExchange (32) : : : +- CometFilter (31) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (30) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (30) : : +- BroadcastExchange (57) : : +- * Project (56) : : +- * SortMergeJoin LeftSemi (55) @@ -59,7 +59,7 @@ TakeOrderedAndProject (93) : : : +- CometSort (47) : : : +- CometExchange (46) : : : +- CometFilter (45) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (44) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (44) : : +- * Sort (54) : : +- * Project (53) : : +- * Filter (52) @@ -82,7 +82,7 @@ TakeOrderedAndProject (93) : : : +- CometProject (71) : : : +- CometBroadcastHashJoin (70) : : : :- CometFilter (68) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (67) : : : +- ReusedExchange (69) : : +- * Sort (80) : : +- * Project (79) @@ -94,7 +94,7 @@ TakeOrderedAndProject (93) +- ReusedExchange (85) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -106,7 +106,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(5) CometScan [native_comet] parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -146,7 +146,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -216,7 +216,7 @@ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ (26) CometColumnarToRow [codegen id : 1] Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(27) CometScan [native_comet] parquet spark_catalog.default.store_sales +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -231,7 +231,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(30) CometScan [native_comet] parquet spark_catalog.default.customer +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -297,7 +297,7 @@ Right keys [1]: [c_customer_sk#23] Join type: LeftSemi Join condition: None -(44) CometScan [native_comet] parquet spark_catalog.default.customer +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -402,7 +402,7 @@ Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] Aggregate Attributes [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42] Results [3]: [c_last_name#36, c_first_name#35, sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42 AS sales#43] -(67) CometScan [native_comet] parquet spark_catalog.default.web_sales +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] Batched: true Location: InMemoryFileIndex [] @@ -528,10 +528,10 @@ BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometProject (96) +- CometFilter (95) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (94) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) -(94) CometScan [native_comet] parquet spark_catalog.default.date_dim +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#37, d_year#67, d_moy#68] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -558,10 +558,10 @@ BroadcastExchange (103) +- * CometColumnarToRow (102) +- CometProject (101) +- CometFilter (100) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (99) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (99) -(99) CometScan [native_comet] parquet spark_catalog.default.date_dim +(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -598,15 +598,15 @@ Subquery:3 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquer :- CometProject (108) : +- CometBroadcastHashJoin (107) : :- CometFilter (105) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (104) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) : +- ReusedExchange (106) +- CometBroadcastExchange (112) +- CometProject (111) +- CometFilter (110) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (109) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (109) -(104) CometScan [native_comet] parquet spark_catalog.default.store_sales +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#69, ss_quantity#70, ss_sales_price#71, ss_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] @@ -630,7 +630,7 @@ Arguments: [ss_customer_sk#69], [c_customer_sk#74], Inner, BuildRight Input [5]: [ss_customer_sk#69, ss_quantity#70, ss_sales_price#71, ss_sold_date_sk#72, c_customer_sk#74] Arguments: [ss_quantity#70, ss_sales_price#71, ss_sold_date_sk#72, c_customer_sk#74], [ss_quantity#70, ss_sales_price#71, ss_sold_date_sk#72, c_customer_sk#74] -(109) CometScan [native_comet] parquet spark_catalog.default.date_dim +(109) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#75, d_year#76] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -708,10 +708,10 @@ BroadcastExchange (128) +- * CometColumnarToRow (127) +- CometProject (126) +- CometFilter (125) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (124) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (124) -(124) CometScan [native_comet] parquet spark_catalog.default.date_dim +(124) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#75, d_year#76] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt index 91415e0e77..667e75dad3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt @@ -21,7 +21,7 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -29,7 +29,7 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -41,7 +41,7 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) @@ -49,14 +49,14 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] InputAdapter WholeStageCodegen (3) Sort [c_customer_sk] @@ -83,7 +83,7 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #13 WholeStageCodegen (1) @@ -91,12 +91,12 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] CometColumnarToRow InputAdapter @@ -109,10 +109,10 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] InputAdapter BroadcastExchange #15 WholeStageCodegen (7) @@ -125,7 +125,7 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] CometSort [c_customer_sk,c_first_name,c_last_name] CometExchange [c_customer_sk] #16 CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter WholeStageCodegen (6) Sort [c_customer_sk] @@ -159,7 +159,7 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [item_sk] #4 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt index 35d1a4166f..df3b1ff6b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_comet] parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_comet] parquet spark_catalog.default.item +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_comet] parquet spark_catalog.default.customer_address +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] -(57) CometScan [native_comet] parquet spark_catalog.default.item +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt index 8444341b4c..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt index 1f2579f6fe..4e088146fe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_comet] parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_comet] parquet spark_catalog.default.item +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_comet] parquet spark_catalog.default.customer_address +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] -(57) CometScan [native_comet] parquet spark_catalog.default.item +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt index 8444341b4c..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt index 82a77d9f4b..c50b3fbc82 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt @@ -20,33 +20,33 @@ TakeOrderedAndProject (45) : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_comet] parquet spark_catalog.default.store (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.item (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -58,7 +58,7 @@ ReadSchema: struct (avg(ctr_total_ret Output [2]: [ctr_customer_sk#14, ctr_total_return#16] Input [5]: [ctr_customer_sk#14, ctr_state#15, ctr_total_return#16, (avg(ctr_total_return) * 1.2)#34, ctr_state#27] -(42) CometScan [native_comet] parquet spark_catalog.default.customer +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [14]: [c_customer_sk#35, c_customer_id#36, c_current_addr_sk#37, c_salutation#38, c_first_name#39, c_last_name#40, c_preferred_cust_flag#41, c_birth_day#42, c_birth_month#43, c_birth_year#44, c_birth_country#45, c_login#46, c_email_address#47, c_last_review_date#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -284,7 +284,7 @@ Join condition: None Output [14]: [ctr_total_return#16, c_customer_id#49, c_current_addr_sk#37, c_salutation#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_day#42, c_birth_month#43, c_birth_year#44, c_birth_country#45, c_login#54, c_email_address#55, c_last_review_date#48] Input [16]: [ctr_customer_sk#14, ctr_total_return#16, c_customer_sk#35, c_customer_id#49, c_current_addr_sk#37, c_salutation#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_day#42, c_birth_month#43, c_birth_year#44, c_birth_country#45, c_login#54, c_email_address#55, c_last_review_date#48] -(49) CometScan [native_comet] parquet spark_catalog.default.customer_address +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#56, ca_state#57] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -327,10 +327,10 @@ BroadcastExchange (61) +- * CometColumnarToRow (60) +- CometProject (59) +- CometFilter (58) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (57) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (57) -(57) CometScan [native_comet] parquet spark_catalog.default.date_dim +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt index c789c4240c..6321e1c947 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -28,15 +28,15 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) @@ -60,7 +60,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk,ca_state] #4 @@ -71,7 +71,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre InputAdapter CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] InputAdapter BroadcastExchange #9 WholeStageCodegen (6) @@ -79,4 +79,4 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt index a87ce617a7..455aac71c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt @@ -20,13 +20,13 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : : : +- BroadcastExchange (33) : : : : +- * HashAggregate (32) : : : : +- * CometColumnarToRow (31) @@ -38,10 +38,10 @@ : : : : :- CometProject (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (19) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (18) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (18) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometFilter (21) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (20) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (20) : : : : +- ReusedExchange (25) : : : +- BroadcastExchange (50) : : : +- * HashAggregate (49) @@ -54,10 +54,10 @@ : : : :- CometProject (41) : : : : +- CometBroadcastHashJoin (40) : : : : :- CometFilter (36) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (35) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (35) : : : : +- CometBroadcastExchange (39) : : : : +- CometFilter (38) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (37) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (37) : : : +- ReusedExchange (42) : : +- BroadcastExchange (66) : : +- * HashAggregate (65) @@ -70,7 +70,7 @@ : : :- CometProject (57) : : : +- CometBroadcastHashJoin (56) : : : :- CometFilter (54) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (53) : : : +- ReusedExchange (55) : : +- ReusedExchange (58) : +- BroadcastExchange (81) @@ -84,7 +84,7 @@ : :- CometProject (72) : : +- CometBroadcastHashJoin (71) : : :- CometFilter (69) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (68) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (68) : : +- ReusedExchange (70) : +- ReusedExchange (73) +- BroadcastExchange (97) @@ -98,12 +98,12 @@ :- CometProject (88) : +- CometBroadcastHashJoin (87) : :- CometFilter (85) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (84) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (84) : +- ReusedExchange (86) +- ReusedExchange (89) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -139,7 +139,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_county#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -187,7 +187,7 @@ Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#12] Results [3]: [ca_county#9, d_year#6, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS store_sales#13] -(18) CometScan [native_comet] parquet spark_catalog.default.store_sales +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -199,7 +199,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16] Condition : isnotnull(ss_addr_sk#14) -(20) CometScan [native_comet] parquet spark_catalog.default.date_dim +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#18, d_year#19, d_qoy#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -269,7 +269,7 @@ Right keys [1]: [ca_county#22] Join type: Inner Join condition: None -(35) CometScan [native_comet] parquet spark_catalog.default.store_sales +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -281,7 +281,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Condition : isnotnull(ss_addr_sk#26) -(37) CometScan [native_comet] parquet spark_catalog.default.date_dim +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#30, d_year#31, d_qoy#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -355,7 +355,7 @@ Join condition: None Output [5]: [ca_county#9, d_year#6, store_sales#13, store_sales#25, store_sales#37] Input [7]: [ca_county#9, d_year#6, store_sales#13, ca_county#22, store_sales#25, ca_county#34, store_sales#37] -(53) CometScan [native_comet] parquet spark_catalog.default.web_sales +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40] Batched: true Location: InMemoryFileIndex [] @@ -425,7 +425,7 @@ Right keys [1]: [ca_county#46] Join type: Inner Join condition: None -(68) CometScan [native_comet] parquet spark_catalog.default.web_sales +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] @@ -499,7 +499,7 @@ Join condition: (CASE WHEN (web_sales#50 > 0.00) THEN (web_sales#62 / web_sales# Output [8]: [ca_county#9, d_year#6, store_sales#13, store_sales#25, store_sales#37, ca_county#46, web_sales#50, web_sales#62] Input [9]: [ca_county#9, d_year#6, store_sales#13, store_sales#25, store_sales#37, ca_county#46, web_sales#50, ca_county#59, web_sales#62] -(84) CometScan [native_comet] parquet spark_catalog.default.web_sales +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65] Batched: true Location: InMemoryFileIndex [] @@ -590,10 +590,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dyn BroadcastExchange (106) +- * CometColumnarToRow (105) +- CometFilter (104) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (103) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_comet] parquet spark_catalog.default.date_dim +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -615,10 +615,10 @@ Subquery:2 Hosting operator id = 18 Hosting Expression = ss_sold_date_sk#16 IN d BroadcastExchange (110) +- * CometColumnarToRow (109) +- CometFilter (108) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (107) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(107) CometScan [native_comet] parquet spark_catalog.default.date_dim +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#18, d_year#19, d_qoy#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -640,10 +640,10 @@ Subquery:3 Hosting operator id = 35 Hosting Expression = ss_sold_date_sk#28 IN d BroadcastExchange (114) +- * CometColumnarToRow (113) +- CometFilter (112) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (111) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (111) -(111) CometScan [native_comet] parquet spark_catalog.default.date_dim +(111) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#30, d_year#31, d_qoy#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt index 4824e1ec5b..945aa41879 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt @@ -25,20 +25,20 @@ WholeStageCodegen (13) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [ca_address_sk,ca_county] #5 CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) @@ -55,17 +55,17 @@ WholeStageCodegen (13) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 InputAdapter BroadcastExchange #10 @@ -83,17 +83,17 @@ WholeStageCodegen (13) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #12 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 InputAdapter BroadcastExchange #14 @@ -111,7 +111,7 @@ WholeStageCodegen (13) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year,d_qoy] #4 ReusedExchange [ca_address_sk,ca_county] #5 @@ -131,7 +131,7 @@ WholeStageCodegen (13) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #9 ReusedExchange [ca_address_sk,ca_county] #5 @@ -151,7 +151,7 @@ WholeStageCodegen (13) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk,d_year,d_qoy] #13 ReusedExchange [ca_address_sk,ca_county] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt index 8e07875bf1..d351df0644 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt @@ -11,11 +11,11 @@ : : +- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- BroadcastExchange (24) : +- * Filter (23) : +- * HashAggregate (22) @@ -26,15 +26,15 @@ : +- CometProject (17) : +- CometBroadcastHashJoin (16) : :- CometFilter (11) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (10) : +- CometBroadcastExchange (15) : +- CometProject (14) : +- CometFilter (13) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (12) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) +- ReusedExchange (27) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -77,7 +77,7 @@ Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5], [cs_ext_disc (9) CometColumnarToRow [codegen id : 4] Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -(10) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -89,7 +89,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Condition : isnotnull(cs_item_sk#7) -(12) CometScan [native_comet] parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -200,10 +200,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt index d92d20d101..32b491805d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt @@ -14,7 +14,7 @@ WholeStageCodegen (5) CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -22,11 +22,11 @@ WholeStageCodegen (5) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) @@ -42,11 +42,11 @@ WholeStageCodegen (5) CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt index 45e659e30b..4c39971dd5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt @@ -17,23 +17,23 @@ TakeOrderedAndProject (67) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- * HashAggregate (45) : +- * CometColumnarToRow (44) : +- CometColumnarExchange (43) @@ -46,7 +46,7 @@ TakeOrderedAndProject (67) : : :- CometProject (34) : : : +- CometBroadcastHashJoin (33) : : : :- CometFilter (31) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (30) : : : +- ReusedExchange (32) : : +- ReusedExchange (35) : +- ReusedExchange (38) @@ -62,13 +62,13 @@ TakeOrderedAndProject (67) : :- CometProject (50) : : +- CometBroadcastHashJoin (49) : : :- CometFilter (47) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (46) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (46) : : +- ReusedExchange (48) : +- ReusedExchange (51) +- ReusedExchange (54) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -80,7 +80,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_manufact_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_category#13, i_manufact_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -208,7 +208,7 @@ Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] Results [2]: [i_manufact_id#12, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#18] -(30) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -280,7 +280,7 @@ Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#21))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#21))#30] Results [2]: [i_manufact_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#21))#30,17,2) AS total_sales#31] -(46) CometScan [native_comet] parquet spark_catalog.default.web_sales +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] @@ -386,10 +386,10 @@ BroadcastExchange (72) +- * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_comet] parquet spark_catalog.default.date_dim +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt index 2ece72af43..bb7182a637 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt @@ -24,7 +24,7 @@ TakeOrderedAndProject [total_sales,i_manufact_id] CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -32,23 +32,23 @@ TakeOrderedAndProject [total_sales,i_manufact_id] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_manufact_id] #6 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #7 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] WholeStageCodegen (4) HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] CometColumnarToRow @@ -65,7 +65,7 @@ TakeOrderedAndProject [total_sales,i_manufact_id] CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -86,7 +86,7 @@ TakeOrderedAndProject [total_sales,i_manufact_id] CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt index ea37bca58f..25339900c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt @@ -15,26 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt index 238061487f..a7c1215d1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt index af814b1761..4db51f2231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt @@ -15,40 +15,40 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (13) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) : : : +- ReusedExchange (14) : : +- BroadcastExchange (25) : : +- * CometColumnarToRow (24) : : +- CometProject (23) : : +- CometBroadcastHashJoin (22) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (20) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) : : +- ReusedExchange (21) : +- BroadcastExchange (33) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,14 +59,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#22] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt index 0d595eb78c..f1fe09fb46 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt index 467ceb2dc5..604be191fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt @@ -18,22 +18,22 @@ TakeOrderedAndProject (32) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) +- CometBroadcastExchange (18) +- CometProject (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,7 +45,7 @@ ReadSchema: struct= 68.00)) A Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#2, 16)) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan [native_comet] parquet spark_catalog.default.inventory +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +71,7 @@ Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#13, cs_sold_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt index 660b5c5de8..7ab646a628 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt @@ -14,11 +14,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -26,11 +26,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [cs_item_sk] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt index 2ee39c4634..3079061df7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt @@ -14,15 +14,15 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : +- CometBroadcastExchange (29) : +- CometHashAggregate (28) : +- CometExchange (27) @@ -32,7 +32,7 @@ : :- CometProject (22) : : +- CometBroadcastHashJoin (21) : : :- CometFilter (19) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- ReusedExchange (23) +- CometBroadcastExchange (42) @@ -44,12 +44,12 @@ :- CometProject (35) : +- CometBroadcastHashJoin (34) : :- CometFilter (32) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (31) : +- ReusedExchange (33) +- ReusedExchange (36) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -89,7 +89,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_comet] parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -131,7 +131,7 @@ Input [3]: [c_last_name#11, c_first_name#10, d_date#5] Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] Functions: [] -(18) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -190,7 +190,7 @@ Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] Right output [3]: [c_last_name#19, c_first_name#18, d_date#16] Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)], LeftSemi, BuildRight -(31) CometScan [native_comet] parquet spark_catalog.default.web_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -276,10 +276,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt index fb925a86b6..c9ecc235c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -23,15 +23,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometExchange [c_last_name,c_first_name,d_date] #7 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 @@ -54,7 +54,7 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt index e55d5dca49..290e333514 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt @@ -17,17 +17,17 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) +- BroadcastExchange (47) +- * Project (46) +- * Filter (45) @@ -43,16 +43,16 @@ : :- CometProject (30) : : +- CometBroadcastHashJoin (29) : : :- CometFilter (27) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (26) : : +- ReusedExchange (28) : +- ReusedExchange (31) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct 0.000000)) -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#31, c_customer_id#32, c_first_name#33, c_last_name#34, c_preferred_cust_flag#35, c_birth_country#36, c_login#37, c_email_address#38] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -226,7 +226,7 @@ Condition : (isnotnull(c_customer_sk#31) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#31, c_customer_id#32, c_first_name#33, c_last_name#34, c_preferred_cust_flag#35, c_birth_country#36, c_login#37, c_email_address#38] Arguments: [c_customer_sk#31, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#36, c_login#43, c_email_address#44], [c_customer_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#32, 16)) AS c_customer_id#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#33, 20)) AS c_first_name#40, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#34, 30)) AS c_last_name#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#35, 1)) AS c_preferred_cust_flag#42, c_birth_country#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#37, 13)) AS c_login#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#38, 50)) AS c_email_address#44] -(23) CometScan [native_comet] parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_customer_sk#45, ss_ext_discount_amt#46, ss_ext_sales_price#47, ss_ext_wholesale_cost#48, ss_ext_list_price#49, ss_sold_date_sk#50] Batched: true Location: InMemoryFileIndex [] @@ -251,7 +251,7 @@ Arguments: [c_customer_sk#31], [ss_customer_sk#45], Inner, BuildRight Input [14]: [c_customer_sk#31, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#36, c_login#43, c_email_address#44, ss_customer_sk#45, ss_ext_discount_amt#46, ss_ext_sales_price#47, ss_ext_wholesale_cost#48, ss_ext_list_price#49, ss_sold_date_sk#50] Arguments: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#36, c_login#43, c_email_address#44, ss_ext_discount_amt#46, ss_ext_sales_price#47, ss_ext_wholesale_cost#48, ss_ext_list_price#49, ss_sold_date_sk#50], [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#36, c_login#43, c_email_address#44, ss_ext_discount_amt#46, ss_ext_sales_price#47, ss_ext_wholesale_cost#48, ss_ext_list_price#49, ss_sold_date_sk#50] -(28) CometScan [native_comet] parquet spark_catalog.default.date_dim +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#52, d_year#53] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -309,7 +309,7 @@ Right keys [1]: [customer_id#58] Join type: Inner Join condition: None -(40) CometScan [native_comet] parquet spark_catalog.default.customer +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#66, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#71, c_login#72, c_email_address#73] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -324,7 +324,7 @@ Condition : (isnotnull(c_customer_sk#66) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#66, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#71, c_login#72, c_email_address#73] Arguments: [c_customer_sk#66, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#71, c_login#78, c_email_address#79], [c_customer_sk#66, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#67, 16)) AS c_customer_id#74, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#68, 20)) AS c_first_name#75, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#69, 30)) AS c_last_name#76, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#70, 1)) AS c_preferred_cust_flag#77, c_birth_country#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#72, 13)) AS c_login#78, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#73, 50)) AS c_email_address#79] -(43) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#80, cs_ext_discount_amt#81, cs_ext_sales_price#82, cs_ext_wholesale_cost#83, cs_ext_list_price#84, cs_sold_date_sk#85] Batched: true Location: InMemoryFileIndex [] @@ -403,7 +403,7 @@ Join condition: None Output [11]: [customer_id#29, year_total#30, customer_id#58, customer_first_name#59, customer_last_name#60, customer_preferred_cust_flag#61, customer_birth_country#62, customer_login#63, customer_email_address#64, year_total#65, year_total#95] Input [12]: [customer_id#29, year_total#30, customer_id#58, customer_first_name#59, customer_last_name#60, customer_preferred_cust_flag#61, customer_birth_country#62, customer_login#63, customer_email_address#64, year_total#65, customer_id#94, year_total#95] -(60) CometScan [native_comet] parquet spark_catalog.default.customer +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#96, c_customer_id#97, c_first_name#98, c_last_name#99, c_preferred_cust_flag#100, c_birth_country#101, c_login#102, c_email_address#103] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -418,7 +418,7 @@ Condition : (isnotnull(c_customer_sk#96) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#96, c_customer_id#97, c_first_name#98, c_last_name#99, c_preferred_cust_flag#100, c_birth_country#101, c_login#102, c_email_address#103] Arguments: [c_customer_sk#96, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#101, c_login#108, c_email_address#109], [c_customer_sk#96, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#97, 16)) AS c_customer_id#104, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#98, 20)) AS c_first_name#105, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#99, 30)) AS c_last_name#106, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#100, 1)) AS c_preferred_cust_flag#107, c_birth_country#101, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#102, 13)) AS c_login#108, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#103, 50)) AS c_email_address#109] -(63) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#110, cs_ext_discount_amt#111, cs_ext_sales_price#112, cs_ext_wholesale_cost#113, cs_ext_list_price#114, cs_sold_date_sk#115] Batched: true Location: InMemoryFileIndex [] @@ -493,7 +493,7 @@ Join condition: (CASE WHEN (year_total#95 > 0.000000) THEN (year_total#124 / yea Output [10]: [customer_id#29, customer_id#58, customer_first_name#59, customer_last_name#60, customer_preferred_cust_flag#61, customer_birth_country#62, customer_login#63, customer_email_address#64, year_total#95, year_total#124] Input [13]: [customer_id#29, year_total#30, customer_id#58, customer_first_name#59, customer_last_name#60, customer_preferred_cust_flag#61, customer_birth_country#62, customer_login#63, customer_email_address#64, year_total#65, year_total#95, customer_id#123, year_total#124] -(79) CometScan [native_comet] parquet spark_catalog.default.customer +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -508,7 +508,7 @@ Condition : (isnotnull(c_customer_sk#125) AND isnotnull(static_invoke(CharVarcha Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] Arguments: [c_customer_sk#125, c_customer_id#133, c_first_name#134, c_last_name#135, c_preferred_cust_flag#136, c_birth_country#130, c_login#137, c_email_address#138], [c_customer_sk#125, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#126, 16)) AS c_customer_id#133, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#127, 20)) AS c_first_name#134, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#128, 30)) AS c_last_name#135, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#129, 1)) AS c_preferred_cust_flag#136, c_birth_country#130, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#131, 13)) AS c_login#137, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#132, 50)) AS c_email_address#138] -(82) CometScan [native_comet] parquet spark_catalog.default.web_sales +(82) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#139, ws_ext_discount_amt#140, ws_ext_sales_price#141, ws_ext_wholesale_cost#142, ws_ext_list_price#143, ws_sold_date_sk#144] Batched: true Location: InMemoryFileIndex [] @@ -587,7 +587,7 @@ Join condition: None Output [11]: [customer_id#29, customer_id#58, customer_first_name#59, customer_last_name#60, customer_preferred_cust_flag#61, customer_birth_country#62, customer_login#63, customer_email_address#64, year_total#95, year_total#124, year_total#154] Input [12]: [customer_id#29, customer_id#58, customer_first_name#59, customer_last_name#60, customer_preferred_cust_flag#61, customer_birth_country#62, customer_login#63, customer_email_address#64, year_total#95, year_total#124, customer_id#153, year_total#154] -(99) CometScan [native_comet] parquet spark_catalog.default.customer +(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#155, c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -602,7 +602,7 @@ Condition : (isnotnull(c_customer_sk#155) AND isnotnull(static_invoke(CharVarcha Input [8]: [c_customer_sk#155, c_customer_id#156, c_first_name#157, c_last_name#158, c_preferred_cust_flag#159, c_birth_country#160, c_login#161, c_email_address#162] Arguments: [c_customer_sk#155, c_customer_id#163, c_first_name#164, c_last_name#165, c_preferred_cust_flag#166, c_birth_country#160, c_login#167, c_email_address#168], [c_customer_sk#155, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#156, 16)) AS c_customer_id#163, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#157, 20)) AS c_first_name#164, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#158, 30)) AS c_last_name#165, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#159, 1)) AS c_preferred_cust_flag#166, c_birth_country#160, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#161, 13)) AS c_login#167, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#162, 50)) AS c_email_address#168] -(102) CometScan [native_comet] parquet spark_catalog.default.web_sales +(102) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#169, ws_ext_discount_amt#170, ws_ext_sales_price#171, ws_ext_wholesale_cost#172, ws_ext_list_price#173, ws_sold_date_sk#174] Batched: true Location: InMemoryFileIndex [] @@ -687,10 +687,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dy BroadcastExchange (122) +- * CometColumnarToRow (121) +- CometFilter (120) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (119) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (119) -(119) CometScan [native_comet] parquet spark_catalog.default.date_dim +(119) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -712,10 +712,10 @@ Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#50 IN d BroadcastExchange (126) +- * CometColumnarToRow (125) +- CometFilter (124) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (123) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (123) -(123) CometScan [native_comet] parquet spark_catalog.default.date_dim +(123) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#52, d_year#53] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt index 2cc1a4a33a..57fa50415c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt @@ -24,20 +24,20 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -55,20 +55,20 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) @@ -87,10 +87,10 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 InputAdapter @@ -110,10 +110,10 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 InputAdapter @@ -134,10 +134,10 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 InputAdapter @@ -157,9 +157,9 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt index 81f388b060..c9374ed00c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt @@ -16,26 +16,26 @@ TakeOrderedAndProject (34) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_comet] parquet spark_catalog.default.item (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) +- CometBroadcastExchange (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -55,7 +55,7 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -87,7 +87,7 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -(12) CometScan [native_comet] parquet spark_catalog.default.warehouse +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#11, w_state#12] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -115,7 +115,7 @@ Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -(18) CometScan [native_comet] parquet spark_catalog.default.item +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -143,7 +143,7 @@ Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -201,10 +201,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dyn BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometFilter (36) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) -(35) CometScan [native_comet] parquet spark_catalog.default.date_dim +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt index 522ed691ed..78502381d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt @@ -19,27 +19,27 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] CometExchange [cr_order_number,cr_item_sk] #4 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometBroadcastExchange [w_warehouse_sk,w_state] #5 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #7 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt index c203d9b28d..71f0293c69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,10 +17,10 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometScan [native_comet] parquet spark_catalog.default.item (4) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -35,7 +35,7 @@ Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_m Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometScan [native_comet] parquet spark_catalog.default.item +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt index b568d3ebc1..bff847188c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] @@ -18,4 +18,4 @@ WholeStageCodegen (1) CometHashAggregate [i_manufact,count] CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometScan [native_comet] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt index c20b668675..ac23993ef3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt @@ -11,17 +11,17 @@ TakeOrderedAndProject (20) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -36,7 +36,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt index c2654e966d..a12475f250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt @@ -14,11 +14,11 @@ TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 CometProject [i_category] [i_item_sk,i_category_id,i_category] CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt index 716bf70e32..78c017dfba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt @@ -11,17 +11,17 @@ TakeOrderedAndProject (20) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.store (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -36,7 +36,7 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(d_day_name#3, 9)) AS d_day_name#4] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometScan [native_comet] parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt index df1ac31ec8..933233db3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt @@ -14,11 +14,11 @@ TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [d_day_name] [d_date_sk,d_day_name] CometFilter [d_date_sk,d_year,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt index 4b2a154ce3..c9bd3f497d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt @@ -26,7 +26,7 @@ : : : +- * CometColumnarToRow (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometSort (35) : : +- CometColumnarExchange (34) : : +- * Project (33) @@ -45,11 +45,11 @@ : +- CometBroadcastExchange (41) : +- CometProject (40) : +- CometFilter (39) - : +- CometScan [native_comet] parquet spark_catalog.default.item (38) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (38) +- ReusedExchange (44) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -204,7 +204,7 @@ Arguments: [rnk#14], [rnk#22], Inner Input [4]: [item_sk#10, rnk#14, item_sk#20, rnk#22] Arguments: [item_sk#10, rnk#14, item_sk#20], [item_sk#10, rnk#14, item_sk#20] -(38) CometScan [native_comet] parquet spark_catalog.default.item +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#23, i_product_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -261,10 +261,10 @@ Subquery:1 Hosting operator id = 9 Hosting Expression = Subquery scalar-subquery +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (49) -(49) CometScan [native_comet] parquet spark_catalog.default.store_sales +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_addr_sk#30, ss_store_sk#31, ss_net_profit#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt index abcb84a0c8..e7519f5fd9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt @@ -37,7 +37,7 @@ WholeStageCodegen (9) InputAdapter CometProject [ss_store_sk,ss_net_profit] CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] CometColumnarToRow InputAdapter @@ -48,7 +48,7 @@ WholeStageCodegen (9) InputAdapter CometProject [ss_item_sk,ss_net_profit] CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] CometSort [item_sk,rnk] CometColumnarExchange [rnk] #5 WholeStageCodegen (8) @@ -74,5 +74,5 @@ WholeStageCodegen (9) CometBroadcastExchange [i_item_sk,i_product_name] #7 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] ReusedExchange [i_item_sk,i_product_name] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt index 307cb6dc30..cf57f11d1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt @@ -17,30 +17,30 @@ TakeOrderedAndProject (39) : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.item (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) +- BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.item (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -52,7 +52,7 @@ ReadSchema: struct= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometScan [native_comet] parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -67,7 +67,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -95,7 +95,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(14) CometScan [native_comet] parquet spark_catalog.default.customer_address +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -123,7 +123,7 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH, Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(20) CometScan [native_comet] parquet spark_catalog.default.date_dim +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -182,10 +182,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_comet] parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt index 707b8a4bda..f515f6562a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (2) CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -24,19 +24,19 @@ WholeStageCodegen (2) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt index 507396b7f7..2029d9b047 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt @@ -25,14 +25,14 @@ : : :- CometBroadcastExchange (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometProject (7) : : +- CometFilter (6) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) :- * Project (53) : +- * Filter (52) : +- Window (51) @@ -53,10 +53,10 @@ : : :- CometBroadcastExchange (32) : : : +- CometProject (31) : : : +- CometFilter (30) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (29) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (33) : +- ReusedExchange (38) +- * Project (78) +- * Filter (77) @@ -78,14 +78,14 @@ : :- CometBroadcastExchange (57) : : +- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (54) : +- CometProject (60) : +- CometFilter (59) - : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (58) +- ReusedExchange (63) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -105,7 +105,7 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometScan [native_comet] parquet spark_catalog.default.web_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -129,7 +129,7 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -212,7 +212,7 @@ Condition : ((return_rank#35 <= 10) OR (currency_rank#36 <= 10)) Output [5]: [web AS channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Input [5]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35, currency_rank#36] -(29) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_net_profit#42, cs_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -232,7 +232,7 @@ Arguments: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, c Input [5]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] Arguments: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] -(33) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -323,7 +323,7 @@ Condition : ((return_rank#70 <= 10) OR (currency_rank#71 <= 10)) Output [5]: [catalog AS channel#72, item#67, return_ratio#68, return_rank#70, currency_rank#71] Input [5]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70, currency_rank#71] -(54) CometScan [native_comet] parquet spark_catalog.default.store_sales +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_net_profit#77, ss_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, Input [5]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] Arguments: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] -(58) CometScan [native_comet] parquet spark_catalog.default.store_returns +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83, sr_returned_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -466,10 +466,10 @@ BroadcastExchange (89) +- * CometColumnarToRow (88) +- CometProject (87) +- CometFilter (86) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (85) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (85) -(85) CometScan [native_comet] parquet spark_catalog.default.date_dim +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt index 6a10ef7a5c..0234327b20 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt @@ -38,7 +38,7 @@ WholeStageCodegen (17) CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) @@ -46,14 +46,14 @@ WholeStageCodegen (17) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] WholeStageCodegen (10) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -84,11 +84,11 @@ WholeStageCodegen (17) CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 WholeStageCodegen (15) Project [item,return_ratio,return_rank,currency_rank] @@ -120,9 +120,9 @@ WholeStageCodegen (17) CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt index 86508f958d..2caf7a8fdc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt @@ -18,18 +18,18 @@ TakeOrderedAndProject (77) : : :- CometUnion (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.store (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) :- * HashAggregate (45) : +- * CometColumnarToRow (44) : +- CometColumnarExchange (43) @@ -42,15 +42,15 @@ TakeOrderedAndProject (77) : : :- CometUnion (31) : : : :- CometProject (27) : : : : +- CometFilter (26) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (25) : : : +- CometProject (30) : : : +- CometFilter (29) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (28) : : +- ReusedExchange (32) : +- CometBroadcastExchange (38) : +- CometProject (37) : +- CometFilter (36) - : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (35) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (35) +- * HashAggregate (70) +- * CometColumnarToRow (69) +- CometColumnarExchange (68) @@ -63,22 +63,22 @@ TakeOrderedAndProject (77) : :- CometUnion (56) : : :- CometProject (48) : : : +- CometFilter (47) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (46) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (46) : : +- CometProject (55) : : +- CometBroadcastHashJoin (54) : : :- CometBroadcastExchange (50) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (49) : : +- CometProject (53) : : +- CometFilter (52) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (51) : +- ReusedExchange (57) +- CometBroadcastExchange (63) +- CometProject (62) +- CometFilter (61) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (60) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (60) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -94,7 +94,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -114,7 +114,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_comet] parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -142,7 +142,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_comet] parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -194,7 +194,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt# Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#35, sum(UnscaledValue(return_amt#10))#36, sum(UnscaledValue(profit#9))#37, sum(UnscaledValue(net_loss#11))#38] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#35,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(return_amt#10))#36,17,2) AS returns#40, (MakeDecimal(sum(UnscaledValue(profit#9))#37,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#38,17,2)) AS profit#41, store channel AS channel#42, concat(store, s_store_id#26) AS id#43] -(25) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#44, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -210,7 +210,7 @@ Condition : isnotnull(cs_catalog_page_sk#44) Input [4]: [cs_catalog_page_sk#44, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47] Arguments: [page_sk#49, date_sk#50, sales_price#51, profit#52, return_amt#53, net_loss#54], [cs_catalog_page_sk#44 AS page_sk#49, cs_sold_date_sk#47 AS date_sk#50, cs_ext_sales_price#45 AS sales_price#51, cs_net_profit#46 AS profit#52, 0.00 AS return_amt#53, 0.00 AS net_loss#54] -(28) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#55, cr_return_amount#56, cr_net_loss#57, cr_returned_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -242,7 +242,7 @@ Arguments: [date_sk#50], [d_date_sk#65], Inner, BuildRight Input [7]: [page_sk#49, date_sk#50, sales_price#51, profit#52, return_amt#53, net_loss#54, d_date_sk#65] Arguments: [page_sk#49, sales_price#51, profit#52, return_amt#53, net_loss#54], [page_sk#49, sales_price#51, profit#52, return_amt#53, net_loss#54] -(35) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -294,7 +294,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#51)), sum(UnscaledValue(return_amt Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#51))#77, sum(UnscaledValue(return_amt#53))#78, sum(UnscaledValue(profit#52))#79, sum(UnscaledValue(net_loss#54))#80] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#51))#77,17,2) AS sales#81, MakeDecimal(sum(UnscaledValue(return_amt#53))#78,17,2) AS returns#82, (MakeDecimal(sum(UnscaledValue(profit#52))#79,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#54))#80,17,2)) AS profit#83, catalog channel AS channel#84, concat(catalog_page, cp_catalog_page_id#68) AS id#85] -(46) CometScan [native_comet] parquet spark_catalog.default.web_sales +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#86, ws_ext_sales_price#87, ws_net_profit#88, ws_sold_date_sk#89] Batched: true Location: InMemoryFileIndex [] @@ -310,7 +310,7 @@ Condition : isnotnull(ws_web_site_sk#86) Input [4]: [ws_web_site_sk#86, ws_ext_sales_price#87, ws_net_profit#88, ws_sold_date_sk#89] Arguments: [wsr_web_site_sk#91, date_sk#92, sales_price#93, profit#94, return_amt#95, net_loss#96], [ws_web_site_sk#86 AS wsr_web_site_sk#91, ws_sold_date_sk#89 AS date_sk#92, ws_ext_sales_price#87 AS sales_price#93, ws_net_profit#88 AS profit#94, 0.00 AS return_amt#95, 0.00 AS net_loss#96] -(49) CometScan [native_comet] parquet spark_catalog.default.web_returns +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#97, wr_order_number#98, wr_return_amt#99, wr_net_loss#100, wr_returned_date_sk#101] Batched: true Location: InMemoryFileIndex [] @@ -321,7 +321,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -137,7 +137,7 @@ Arguments: hashpartitioning(item_sk#11, d_date#6, 5), ENSURE_REQUIREMENTS, Comet Input [3]: [item_sk#11, d_date#6, cume_sales#13] Arguments: [item_sk#11, d_date#6, cume_sales#13], [item_sk#11 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#14, ss_sales_price#15, ss_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -251,10 +251,10 @@ BroadcastExchange (50) +- * CometColumnarToRow (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_comet] parquet spark_catalog.default.date_dim +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt index 7e0bed97b2..4e5176325d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt @@ -33,7 +33,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) @@ -41,11 +41,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] CometColumnarExchange [item_sk,d_date] #7 WholeStageCodegen (8) @@ -69,6 +69,6 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt index f331bea3fa..79346703cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt @@ -11,17 +11,17 @@ TakeOrderedAndProject (20) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -36,7 +36,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt index 8ddb4e626e..57a64fea43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt @@ -14,11 +14,11 @@ TakeOrderedAndProject [d_year,ext_price,brand_id,brand] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt index 2cdac08e0d..95f9bd5ac0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt @@ -19,20 +19,20 @@ TakeOrderedAndProject (31) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -47,7 +47,7 @@ Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -72,7 +72,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -100,7 +100,7 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -182,10 +182,10 @@ BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(32) CometScan [native_comet] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt index b3e572b4be..56fe8ae013 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt @@ -26,10 +26,10 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manufact_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -37,11 +37,11 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [d_date_sk,d_qoy] #5 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [s_store_sk] #6 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt index f51c363985..6ebeb92145 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt @@ -29,39 +29,39 @@ TakeOrderedAndProject (60) : : : : : : :- CometUnion (7) : : : : : : : :- CometProject (3) : : : : : : : : +- CometFilter (2) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : +- CometProject (6) : : : : : : : +- CometFilter (5) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (4) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (4) : : : : : : +- CometBroadcastExchange (11) : : : : : : +- CometProject (10) : : : : : : +- CometFilter (9) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) : : : : : +- CometBroadcastExchange (17) : : : : : +- CometProject (16) : : : : : +- CometFilter (15) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : : : : +- CometBroadcastExchange (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (30) : : : +- CometFilter (29) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) : : +- CometBroadcastExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) : +- CometBroadcastExchange (42) : +- CometProject (41) : +- CometFilter (40) - : +- CometScan [native_comet] parquet spark_catalog.default.store (39) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) +- CometBroadcastExchange (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (45) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -77,7 +77,7 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Arguments: [sold_date_sk#5, customer_sk#6, item_sk#7], [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] -(4) CometScan [native_comet] parquet spark_catalog.default.web_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -97,7 +97,7 @@ Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] -(8) CometScan [native_comet] parquet spark_catalog.default.item +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_class#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -125,7 +125,7 @@ Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -153,7 +153,7 @@ Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] Arguments: [customer_sk#6], [customer_sk#6] -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#20, c_current_addr_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -191,7 +191,7 @@ Input [2]: [c_customer_sk#20, c_current_addr_sk#21] Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] Functions: [] -(28) CometScan [native_comet] parquet spark_catalog.default.store_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] @@ -216,7 +216,7 @@ Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -(33) CometScan [native_comet] parquet spark_catalog.default.customer_address +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -244,7 +244,7 @@ Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -(39) CometScan [native_comet] parquet spark_catalog.default.store +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_county#30, s_state#31] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -272,7 +272,7 @@ Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildR Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -(45) CometScan [native_comet] parquet spark_catalog.default.date_dim +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -356,10 +356,10 @@ BroadcastExchange (65) +- * CometColumnarToRow (64) +- CometProject (63) +- CometFilter (62) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (61) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (61) -(61) CometScan [native_comet] parquet spark_catalog.default.date_dim +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -388,10 +388,10 @@ BroadcastExchange (70) +- * CometColumnarToRow (69) +- CometProject (68) +- CometFilter (67) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (66) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) -(66) CometScan [native_comet] parquet spark_catalog.default.date_dim +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -424,10 +424,10 @@ Subquery:6 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (74) +- CometProject (73) +- CometFilter (72) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (71) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (71) -(71) CometScan [native_comet] parquet spark_catalog.default.date_dim +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#48, d_year#49, d_moy#50] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -466,10 +466,10 @@ Subquery:7 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (81) +- CometProject (80) +- CometFilter (79) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (78) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (78) -(78) CometScan [native_comet] parquet spark_catalog.default.date_dim +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#52, d_year#53, d_moy#54] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt index 77454d5263..eb1094ebbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt @@ -34,7 +34,7 @@ TakeOrderedAndProject [segment,num_customers,segment_base] CometUnion [sold_date_sk,customer_sk,item_sk] CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -42,25 +42,25 @@ TakeOrderedAndProject [segment,num_customers,segment_base] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [i_item_sk] #5 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #9 WholeStageCodegen (1) @@ -70,7 +70,7 @@ TakeOrderedAndProject [segment,num_customers,segment_base] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [(d_month_seq + 1)] #3 ReusedSubquery [(d_month_seq + 3)] #4 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] Subquery #3 WholeStageCodegen (1) CometColumnarToRow @@ -80,7 +80,7 @@ TakeOrderedAndProject [segment,num_customers,segment_base] CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #4 WholeStageCodegen (1) CometColumnarToRow @@ -90,20 +90,20 @@ TakeOrderedAndProject [segment,num_customers,segment_base] CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] CometBroadcastExchange [s_county,s_state] #13 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [(d_month_seq + 1)] #3 ReusedSubquery [(d_month_seq + 3)] #4 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] ReusedSubquery [(d_month_seq + 1)] #3 ReusedSubquery [(d_month_seq + 3)] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt index b9359c2935..cd63a353c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt @@ -11,17 +11,17 @@ TakeOrderedAndProject (20) : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -36,7 +36,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt index 21cae37e8a..5c446762f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt @@ -14,11 +14,11 @@ TakeOrderedAndProject [ext_price,brand_id,brand] CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt index b66afa3c9b..63aeb8696a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt @@ -17,24 +17,24 @@ TakeOrderedAndProject (68) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- * HashAggregate (46) : +- * CometColumnarToRow (45) : +- CometColumnarExchange (44) @@ -47,7 +47,7 @@ TakeOrderedAndProject (68) : : :- CometProject (35) : : : +- CometBroadcastHashJoin (34) : : : :- CometFilter (32) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (31) : : : +- ReusedExchange (33) : : +- ReusedExchange (36) : +- ReusedExchange (39) @@ -63,13 +63,13 @@ TakeOrderedAndProject (68) : :- CometProject (51) : : +- CometBroadcastHashJoin (50) : : :- CometFilter (48) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (47) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) : : +- ReusedExchange (49) : +- ReusedExchange (52) +- ReusedExchange (55) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -81,7 +81,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_color#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -212,7 +212,7 @@ Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#19] Results [2]: [i_item_id#16 AS i_item_id#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#19,17,2) AS total_sales#21] -(31) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#22, cs_item_sk#23, cs_ext_sales_price#24, cs_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -284,7 +284,7 @@ Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#24))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#24))#33] Results [2]: [i_item_id#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#24))#33,17,2) AS total_sales#34] -(47) CometScan [native_comet] parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#35, ws_bill_addr_sk#36, ws_ext_sales_price#37, ws_sold_date_sk#38] Batched: true Location: InMemoryFileIndex [] @@ -390,10 +390,10 @@ BroadcastExchange (73) +- * CometColumnarToRow (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (69) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_comet] parquet spark_catalog.default.date_dim +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt index 00184e063c..a96918865c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt @@ -24,7 +24,7 @@ TakeOrderedAndProject [total_sales,i_item_id] CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -32,24 +32,24 @@ TakeOrderedAndProject [total_sales,i_item_id] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_id,i_color] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] WholeStageCodegen (4) HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] CometColumnarToRow @@ -66,7 +66,7 @@ TakeOrderedAndProject [total_sales,i_item_id] CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -87,7 +87,7 @@ TakeOrderedAndProject [total_sales,i_item_id] CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt index 0e4276b9b0..82fc63504a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt @@ -25,16 +25,16 @@ TakeOrderedAndProject (51) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (40) : +- * Project (39) : +- Window (38) @@ -52,7 +52,7 @@ TakeOrderedAndProject (51) +- ReusedExchange (43) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -67,7 +67,7 @@ Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCode Input [3]: [i_item_sk#1, i_brand#2, i_category#3] Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -92,7 +92,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -116,7 +116,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_comet] parquet spark_catalog.default.call_center +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -283,10 +283,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (55) +- * CometColumnarToRow (54) +- CometFilter (53) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (52) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) -(52) CometScan [native_comet] parquet spark_catalog.default.date_dim +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt index 9ccadcfecf..8729139dee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt @@ -34,23 +34,23 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt index babe1ffc6c..f6297c4b7c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt @@ -15,16 +15,16 @@ TakeOrderedAndProject (57) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- BroadcastExchange (37) : +- * Filter (36) @@ -38,7 +38,7 @@ TakeOrderedAndProject (57) : :- CometProject (27) : : +- CometBroadcastHashJoin (26) : : :- CometFilter (24) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (23) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (23) : : +- ReusedExchange (25) : +- ReusedExchange (28) +- BroadcastExchange (54) @@ -53,12 +53,12 @@ TakeOrderedAndProject (57) :- CometProject (44) : +- CometBroadcastHashJoin (43) : :- CometFilter (41) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : +- ReusedExchange (42) +- ReusedExchange (45) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -162,7 +162,7 @@ Results [2]: [i_item_id#7 AS item_id#14, MakeDecimal(sum(UnscaledValue(ss_ext_sa Input [2]: [item_id#14, ss_item_rev#15] Condition : isnotnull(ss_item_rev#15) -(23) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -240,7 +240,7 @@ Join condition: ((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * cs_item_rev# Output [3]: [item_id#14, ss_item_rev#15, cs_item_rev#27] Input [4]: [item_id#14, ss_item_rev#15, item_id#26, cs_item_rev#27] -(40) CometScan [native_comet] parquet spark_catalog.default.web_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] Batched: true Location: InMemoryFileIndex [] @@ -330,14 +330,14 @@ BroadcastExchange (67) +- CometProject (65) +- CometBroadcastHashJoin (64) :- CometFilter (59) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) +- CometBroadcastExchange (63) +- CometProject (62) +- CometFilter (61) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (60) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (60) -(58) CometScan [native_comet] parquet spark_catalog.default.date_dim +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -348,7 +348,7 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(60) CometScan [native_comet] parquet spark_catalog.default.date_dim +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#10, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -389,10 +389,10 @@ Subquery:3 Hosting operator id = 60 Hosting Expression = Subquery scalar-subquer * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_comet] parquet spark_catalog.default.date_dim +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#47, d_week_seq#48] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt index 2766644a37..5880c2033c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt @@ -18,7 +18,7 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -27,28 +27,28 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] CometFilter [d_date,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 InputAdapter BroadcastExchange #6 @@ -67,7 +67,7 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #4 ReusedExchange [d_date_sk] #5 @@ -88,7 +88,7 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #4 ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt index 6ac2e3f092..30e5a3aa65 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt @@ -14,21 +14,21 @@ TakeOrderedAndProject (48) : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- BroadcastExchange (18) : : +- * CometColumnarToRow (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) : +- BroadcastExchange (25) : +- * CometColumnarToRow (24) : +- CometProject (23) : +- CometFilter (22) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (21) +- BroadcastExchange (45) +- * Project (44) +- * BroadcastHashJoin Inner BuildRight (43) @@ -41,15 +41,15 @@ TakeOrderedAndProject (48) : +- * CometColumnarToRow (34) : +- CometProject (33) : +- CometFilter (32) - : +- CometScan [native_comet] parquet spark_catalog.default.store (31) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (31) +- BroadcastExchange (42) +- * CometColumnarToRow (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_s Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))#28] Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END))#22,17,2) AS sun_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END))#23,17,2) AS mon_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END))#24,17,2) AS tue_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END))#25,17,2) AS wed_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END))#26,17,2) AS thu_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END))#27,17,2) AS fri_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))#28,17,2) AS sat_sales#35] -(14) CometScan [native_comet] parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#36, s_store_id#37, s_store_name#38] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -145,7 +145,7 @@ Join condition: None Output [10]: [d_week_seq#5, sun_sales#29, mon_sales#30, tue_sales#31, wed_sales#32, thu_sales#33, fri_sales#34, sat_sales#35, s_store_id#39, s_store_name#38] Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#29, mon_sales#30, tue_sales#31, wed_sales#32, thu_sales#33, fri_sales#34, sat_sales#35, s_store_sk#36, s_store_id#39, s_store_name#38] -(21) CometScan [native_comet] parquet spark_catalog.default.date_dim +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#40, d_week_seq#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -190,7 +190,7 @@ Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday ) THEN ss_ Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday ) THEN ss_sales_price#62 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday ) THEN ss_sales_price#62 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday ) THEN ss_sales_price#62 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN ss_sales_price#62 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN ss_sales_price#62 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday ) THEN ss_sales_price#62 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN ss_sales_price#62 END))#28] Results [9]: [d_week_seq#52, ss_store_sk#53, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Sunday ) THEN ss_sales_price#62 END))#22,17,2) AS sun_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Monday ) THEN ss_sales_price#62 END))#23,17,2) AS mon_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Tuesday ) THEN ss_sales_price#62 END))#24,17,2) AS tue_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Wednesday) THEN ss_sales_price#62 END))#25,17,2) AS wed_sales#66, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Thursday ) THEN ss_sales_price#62 END))#26,17,2) AS thu_sales#67, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Friday ) THEN ss_sales_price#62 END))#27,17,2) AS fri_sales#68, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#61 = Saturday ) THEN ss_sales_price#62 END))#28,17,2) AS sat_sales#69] -(31) CometScan [native_comet] parquet spark_catalog.default.store +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#70, s_store_id#71] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -222,7 +222,7 @@ Join condition: None Output [9]: [d_week_seq#52, sun_sales#63, mon_sales#64, tue_sales#65, wed_sales#66, thu_sales#67, fri_sales#68, sat_sales#69, s_store_id#72] Input [11]: [d_week_seq#52, ss_store_sk#53, sun_sales#63, mon_sales#64, tue_sales#65, wed_sales#66, thu_sales#67, fri_sales#68, sat_sales#69, s_store_sk#70, s_store_id#72] -(38) CometScan [native_comet] parquet spark_catalog.default.date_dim +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#73, d_week_seq#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt index deca8f9636..76738a192c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) @@ -29,7 +29,7 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s InputAdapter CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] InputAdapter BroadcastExchange #4 WholeStageCodegen (3) @@ -37,7 +37,7 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s InputAdapter CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (7) @@ -56,7 +56,7 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s InputAdapter CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] InputAdapter BroadcastExchange #7 WholeStageCodegen (6) @@ -64,4 +64,4 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s InputAdapter CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt index e919aaee71..b2bdb2e5df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt @@ -16,23 +16,23 @@ TakeOrderedAndProject (44) : : : +- CometBroadcastHashJoin (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) : : : +- CometBroadcastExchange (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) : : +- CometBroadcastExchange (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) +- BroadcastExchange (36) +- * Project (35) +- * BroadcastHashJoin Inner BuildRight (34) :- * CometColumnarToRow (23) : +- CometFilter (22) - : +- CometScan [native_comet] parquet spark_catalog.default.item (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (21) +- BroadcastExchange (33) +- * Filter (32) +- * HashAggregate (31) @@ -42,10 +42,10 @@ TakeOrderedAndProject (44) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.item (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) -(1) CometScan [native_comet] parquet spark_catalog.default.customer_address +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -60,7 +60,7 @@ Condition : isnotnull(ca_address_sk#1) Input [2]: [ca_address_sk#1, ca_state#2] Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#2, 2)) AS ca_state#3] -(4) CometScan [native_comet] parquet spark_catalog.default.customer +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -84,7 +84,7 @@ Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(9) CometScan [native_comet] parquet spark_catalog.default.store_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -109,7 +109,7 @@ Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -140,7 +140,7 @@ Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] (20) CometColumnarToRow [codegen id : 4] Input [2]: [ca_state#3, ss_item_sk#6] -(21) CometScan [native_comet] parquet spark_catalog.default.item +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -154,7 +154,7 @@ Condition : ((isnotnull(i_current_price#15) AND isnotnull(i_category#16)) AND is (23) CometColumnarToRow [codegen id : 3] Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] -(24) CometScan [native_comet] parquet spark_catalog.default.item +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_current_price#17, i_category#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -260,10 +260,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (45) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_comet] parquet spark_catalog.default.date_dim +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -294,10 +294,10 @@ Subquery:3 Hosting operator id = 45 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (50) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_comet] parquet spark_catalog.default.date_dim +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#31, d_year#32, d_moy#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt index 4454853146..6445e533dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt @@ -19,13 +19,13 @@ TakeOrderedAndProject [cnt,state] CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -34,7 +34,7 @@ TakeOrderedAndProject [cnt,state] CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] Subquery #2 WholeStageCodegen (1) CometColumnarToRow @@ -44,12 +44,12 @@ TakeOrderedAndProject [cnt,state] CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 InputAdapter BroadcastExchange #7 @@ -59,7 +59,7 @@ TakeOrderedAndProject [cnt,state] CometColumnarToRow InputAdapter CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] InputAdapter BroadcastExchange #8 WholeStageCodegen (2) @@ -74,4 +74,4 @@ TakeOrderedAndProject [cnt,state] InputAdapter CometProject [i_category] [i_category,i_current_price] CometFilter [i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt index 3032613e62..11b7180ad5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt @@ -17,24 +17,24 @@ TakeOrderedAndProject (68) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- * HashAggregate (46) : +- * CometColumnarToRow (45) : +- CometColumnarExchange (44) @@ -47,7 +47,7 @@ TakeOrderedAndProject (68) : : :- CometProject (35) : : : +- CometBroadcastHashJoin (34) : : : :- CometFilter (32) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (31) : : : +- ReusedExchange (33) : : +- ReusedExchange (36) : +- ReusedExchange (39) @@ -63,13 +63,13 @@ TakeOrderedAndProject (68) : :- CometProject (51) : : +- CometBroadcastHashJoin (50) : : :- CometFilter (48) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (47) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) : : +- ReusedExchange (49) : +- ReusedExchange (52) +- ReusedExchange (55) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -81,7 +81,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_category#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -213,7 +213,7 @@ Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#19] Results [2]: [i_item_id#16 AS i_item_id#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#19,17,2) AS total_sales#21] -(31) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#22, cs_item_sk#23, cs_ext_sales_price#24, cs_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -285,7 +285,7 @@ Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#24))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#24))#33] Results [2]: [i_item_id#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#24))#33,17,2) AS total_sales#34] -(47) CometScan [native_comet] parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#35, ws_bill_addr_sk#36, ws_ext_sales_price#37, ws_sold_date_sk#38] Batched: true Location: InMemoryFileIndex [] @@ -391,10 +391,10 @@ BroadcastExchange (73) +- * CometColumnarToRow (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (69) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_comet] parquet spark_catalog.default.date_dim +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt index a0e2b3ad5b..54b8aec25d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt @@ -24,7 +24,7 @@ TakeOrderedAndProject [i_item_id,total_sales] CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -32,24 +32,24 @@ TakeOrderedAndProject [i_item_id,total_sales] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_id,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] WholeStageCodegen (4) HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] CometColumnarToRow @@ -66,7 +66,7 @@ TakeOrderedAndProject [i_item_id,total_sales] CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -87,7 +87,7 @@ TakeOrderedAndProject [i_item_id,total_sales] CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt index f9ddd7e9e8..4bdd124370 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt @@ -19,30 +19,30 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.promotion (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.item (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) +- BroadcastExchange (65) +- * HashAggregate (64) +- * CometColumnarToRow (63) @@ -60,7 +60,7 @@ : : : :- CometProject (47) : : : : +- CometBroadcastHashJoin (46) : : : : :- CometFilter (44) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (43) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (43) : : : : +- ReusedExchange (45) : : : +- ReusedExchange (48) : : +- ReusedExchange (51) @@ -68,7 +68,7 @@ +- ReusedExchange (57) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -80,7 +80,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(6) CometScan [native_comet] parquet spark_catalog.default.date_dim +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Join condition: None Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#13] Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#13] -(21) CometScan [native_comet] parquet spark_catalog.default.item +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -175,7 +175,7 @@ Join condition: None Output [7]: [s_store_name#2, ss_store_sk#4, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#19] Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#13, i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#19] -(28) CometScan [native_comet] parquet spark_catalog.default.store_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#20, ss_store_sk#21, ss_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -273,10 +273,10 @@ BroadcastExchange (51) +- * CometColumnarToRow (50) +- CometProject (49) +- CometFilter (48) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (47) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) -(47) CometScan [native_comet] parquet spark_catalog.default.date_dim +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt index 18fff79ba4..6396b9102d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt @@ -9,7 +9,7 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #1 WholeStageCodegen (2) @@ -25,7 +25,7 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -33,11 +33,11 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -45,7 +45,7 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes InputAdapter CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] InputAdapter BroadcastExchange #6 WholeStageCodegen (6) @@ -67,6 +67,6 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt index b2f6afe88a..27c9f5f4c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt @@ -19,22 +19,22 @@ TakeOrderedAndProject (55) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.ship_mode (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) +- * HashAggregate (49) +- * CometColumnarToRow (48) +- CometColumnarExchange (47) @@ -49,14 +49,14 @@ TakeOrderedAndProject (55) : : :- CometProject (35) : : : +- CometBroadcastHashJoin (34) : : : :- CometFilter (32) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (31) : : : +- ReusedExchange (33) : : +- ReusedExchange (36) : +- ReusedExchange (39) +- ReusedExchange (42) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -68,7 +68,7 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -105,7 +105,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ Right keys [1]: [ws_bill_customer_sk#10] Join type: LeftAnti Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -173,7 +173,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(28) CometScan [native_comet] parquet spark_catalog.default.customer_address +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -205,7 +205,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#2] Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#18] -(35) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -269,10 +269,10 @@ BroadcastExchange (51) +- * CometColumnarToRow (50) +- CometProject (49) +- CometFilter (48) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (47) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) -(47) CometScan [native_comet] parquet spark_catalog.default.date_dim +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt index c68549d28f..68bf32c40b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -29,11 +29,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -41,7 +41,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -51,7 +51,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -61,7 +61,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -69,4 +69,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt index 0dde419f66..ad0e3be68c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt @@ -14,26 +14,26 @@ TakeOrderedAndProject (32) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) +- CometBroadcastExchange (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_comet] parquet spark_catalog.default.promotion (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -45,7 +45,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -94,7 +94,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 5] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_comet] parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -108,7 +108,7 @@ Condition : isnotnull(s_store_sk#7) (12) CometColumnarToRow [codegen id : 4] Input [3]: [s_store_sk#7, s_county#8, s_state#9] -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -120,7 +120,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -287,10 +287,10 @@ BroadcastExchange (55) +- * CometColumnarToRow (54) +- CometProject (53) +- CometFilter (52) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (51) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) -(51) CometScan [native_comet] parquet spark_catalog.default.date_dim +(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt index 4d5eda4c9e..5ec4e3874a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt @@ -23,7 +23,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -31,11 +31,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) @@ -44,7 +44,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) @@ -68,10 +68,10 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_net_profit,ss_sold_date_sk,s_state] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt index 50d65c0ec1..a38f8dd5a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt @@ -14,33 +14,33 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) : : :- CometFilter (6) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (5) : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (13) : : +- ReusedExchange (15) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (19) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (20) +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.time_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -59,7 +59,7 @@ Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, s Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(5) CometScan [native_comet] parquet spark_catalog.default.web_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +71,7 @@ ReadSchema: struct date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_comet] parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_comet] parquet spark_catalog.default.date_dim +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt index a89be99fc8..c770348a8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt index 6560be9c14..87b805556e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt @@ -15,26 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct= 1) AND (cnt#17 <= 5)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt index 3c2f1f34f6..5d798743dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt index 4ce14c3fce..f1dc628799 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt @@ -17,13 +17,13 @@ TakeOrderedAndProject (79) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- BroadcastExchange (38) : : +- * HashAggregate (37) : : +- * CometColumnarToRow (36) @@ -36,13 +36,13 @@ TakeOrderedAndProject (79) : : : +- CometBroadcastHashJoin (26) : : : :- CometProject (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (25) : : : +- CometFilter (24) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : +- CometBroadcastExchange (30) : : +- CometFilter (29) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (28) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (28) : +- BroadcastExchange (57) : +- * Filter (56) : +- * HashAggregate (55) @@ -56,10 +56,10 @@ TakeOrderedAndProject (79) : : +- CometBroadcastHashJoin (46) : : :- CometProject (42) : : : +- CometFilter (41) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (40) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (40) : : +- CometBroadcastExchange (45) : : +- CometFilter (44) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (43) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) : +- ReusedExchange (48) +- BroadcastExchange (76) +- * HashAggregate (75) @@ -73,14 +73,14 @@ TakeOrderedAndProject (79) : +- CometBroadcastHashJoin (66) : :- CometProject (62) : : +- CometFilter (61) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (60) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (60) : +- CometBroadcastExchange (65) : +- CometFilter (64) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (63) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (63) +- ReusedExchange (68) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -95,7 +95,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharC Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -120,7 +120,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -172,7 +172,7 @@ Results [2]: [c_customer_id#5 AS customer_id#17, MakeDecimal(sum(UnscaledValue(s Input [2]: [customer_id#17, year_total#18] Condition : (isnotnull(year_total#18) AND (year_total#18 > 0.00)) -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -187,7 +187,7 @@ Condition : (isnotnull(c_customer_sk#19) AND isnotnull(static_invoke(CharVarchar Input [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22] Arguments: [c_customer_sk#19, c_customer_id#23, c_first_name#24, c_last_name#25], [c_customer_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#20, 16)) AS c_customer_id#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#21, 20)) AS c_first_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#22, 30)) AS c_last_name#25] -(23) CometScan [native_comet] parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#26, ss_net_paid#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -212,7 +212,7 @@ Arguments: [c_customer_sk#19], [ss_customer_sk#26], Inner, BuildRight Input [7]: [c_customer_sk#19, c_customer_id#23, c_first_name#24, c_last_name#25, ss_customer_sk#26, ss_net_paid#27, ss_sold_date_sk#28] Arguments: [c_customer_id#23, c_first_name#24, c_last_name#25, ss_net_paid#27, ss_sold_date_sk#28], [c_customer_id#23, c_first_name#24, c_last_name#25, ss_net_paid#27, ss_sold_date_sk#28] -(28) CometScan [native_comet] parquet spark_catalog.default.date_dim +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#30, d_year#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -270,7 +270,7 @@ Right keys [1]: [customer_id#34] Join type: Inner Join condition: None -(40) CometScan [native_comet] parquet spark_catalog.default.customer +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -285,7 +285,7 @@ Condition : (isnotnull(c_customer_sk#38) AND isnotnull(static_invoke(CharVarchar Input [4]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41] Arguments: [c_customer_sk#38, c_customer_id#42, c_first_name#43, c_last_name#44], [c_customer_sk#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#39, 16)) AS c_customer_id#42, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#40, 20)) AS c_first_name#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#41, 30)) AS c_last_name#44] -(43) CometScan [native_comet] parquet spark_catalog.default.web_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#45, ws_net_paid#46, ws_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -364,7 +364,7 @@ Join condition: None Output [7]: [customer_id#17, year_total#18, customer_id#34, customer_first_name#35, customer_last_name#36, year_total#37, year_total#55] Input [8]: [customer_id#17, year_total#18, customer_id#34, customer_first_name#35, customer_last_name#36, year_total#37, customer_id#54, year_total#55] -(60) CometScan [native_comet] parquet spark_catalog.default.customer +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#56, c_customer_id#57, c_first_name#58, c_last_name#59] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -379,7 +379,7 @@ Condition : (isnotnull(c_customer_sk#56) AND isnotnull(static_invoke(CharVarchar Input [4]: [c_customer_sk#56, c_customer_id#57, c_first_name#58, c_last_name#59] Arguments: [c_customer_sk#56, c_customer_id#60, c_first_name#61, c_last_name#62], [c_customer_sk#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#57, 16)) AS c_customer_id#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#58, 20)) AS c_first_name#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#59, 30)) AS c_last_name#62] -(63) CometScan [native_comet] parquet spark_catalog.default.web_sales +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65] Batched: true Location: InMemoryFileIndex [] @@ -464,10 +464,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (83) +- * CometColumnarToRow (82) +- CometFilter (81) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (80) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (80) -(80) CometScan [native_comet] parquet spark_catalog.default.date_dim +(80) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -489,10 +489,10 @@ Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#28 IN d BroadcastExchange (87) +- * CometColumnarToRow (86) +- CometFilter (85) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (84) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (84) -(84) CometScan [native_comet] parquet spark_catalog.default.date_dim +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#30, d_year#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt index 0c917184b7..5cdabecc4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt @@ -20,20 +20,20 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -51,20 +51,20 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) @@ -83,10 +83,10 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 InputAdapter @@ -106,9 +106,9 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt index ccc0ec8a7e..3e706c643c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt @@ -24,19 +24,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -46,14 +46,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -63,14 +63,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (124) +- CometColumnarExchange (123) +- * Filter (122) @@ -92,11 +92,11 @@ : : :- CometProject (73) : : : +- CometBroadcastHashJoin (72) : : : :- CometFilter (70) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (69) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (69) : : : +- ReusedExchange (71) : : +- CometBroadcastExchange (76) : : +- CometFilter (75) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (74) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) : +- CometSort (82) : +- ReusedExchange (81) :- CometProject (98) @@ -108,7 +108,7 @@ : : :- CometProject (89) : : : +- CometBroadcastHashJoin (88) : : : :- CometFilter (86) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (85) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (85) : : : +- ReusedExchange (87) : : +- ReusedExchange (90) : +- CometSort (96) @@ -122,14 +122,14 @@ : :- CometProject (103) : : +- CometBroadcastHashJoin (102) : : :- CometFilter (100) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (99) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (99) : : +- ReusedExchange (101) : +- ReusedExchange (104) +- CometSort (110) +- ReusedExchange (109) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -141,7 +141,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -86,7 +86,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometScan [native_comet] parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -114,7 +114,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] -(15) CometScan [native_comet] parquet spark_catalog.default.customer_address +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [1]: [ca_zip#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -128,7 +128,7 @@ Condition : (substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip Input [1]: [ca_zip#12] Arguments: [ca_zip#13], [substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#12, 10)), 1, 5) AS ca_zip#13] -(18) CometScan [native_comet] parquet spark_catalog.default.customer_address +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#14, ca_zip#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -143,7 +143,7 @@ Condition : isnotnull(ca_address_sk#14) Input [2]: [ca_address_sk#14, ca_zip#15] Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#15, 10)) AS ca_zip#16] -(21) CometScan [native_comet] parquet spark_catalog.default.customer +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -264,10 +264,10 @@ BroadcastExchange (50) +- * CometColumnarToRow (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_comet] parquet spark_catalog.default.date_dim +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt index 854a2269f7..78a2be6579 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt @@ -15,7 +15,7 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -23,15 +23,15 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometExchange [ca_zip] #6 @@ -39,7 +39,7 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] CometBroadcastExchange [ca_zip] #7 CometProject [ca_zip] CometFilter [ca_zip,cnt] @@ -50,8 +50,8 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [c_current_addr_sk] #9 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt index 549824ea3e..a5afc430cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt @@ -24,28 +24,28 @@ TakeOrderedAndProject (109) : : : : : :- CometSort (4) : : : : : : +- CometExchange (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : +- CometSort (9) : : : : : +- CometExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) : : : : +- CometBroadcastExchange (15) : : : : +- CometProject (14) : : : : +- CometFilter (13) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (12) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) : : : +- CometBroadcastExchange (21) : : : +- CometProject (20) : : : +- CometFilter (19) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) : : +- CometBroadcastExchange (27) : : +- CometProject (26) : : +- CometFilter (25) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (24) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) : +- CometBroadcastExchange (33) : +- CometProject (32) : +- CometFilter (31) - : +- CometScan [native_comet] parquet spark_catalog.default.promotion (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) :- * HashAggregate (71) : +- * CometColumnarToRow (70) : +- CometColumnarExchange (69) @@ -64,17 +64,17 @@ TakeOrderedAndProject (109) : : : : : :- CometSort (44) : : : : : : +- CometExchange (43) : : : : : : +- CometFilter (42) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (41) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (41) : : : : : +- CometSort (49) : : : : : +- CometExchange (48) : : : : : +- CometProject (47) : : : : : +- CometFilter (46) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (45) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (45) : : : : +- ReusedExchange (52) : : : +- CometBroadcastExchange (58) : : : +- CometProject (57) : : : +- CometFilter (56) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (55) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (55) : : +- ReusedExchange (61) : +- ReusedExchange (64) +- * HashAggregate (102) @@ -95,22 +95,22 @@ TakeOrderedAndProject (109) : : : : :- CometSort (75) : : : : : +- CometExchange (74) : : : : : +- CometFilter (73) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (72) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (72) : : : : +- CometSort (80) : : : : +- CometExchange (79) : : : : +- CometProject (78) : : : : +- CometFilter (77) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (76) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (76) : : : +- ReusedExchange (83) : : +- CometBroadcastExchange (89) : : +- CometProject (88) : : +- CometFilter (87) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_site (86) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (86) : +- ReusedExchange (92) +- ReusedExchange (95) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -130,7 +130,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -162,7 +162,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#1 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -(12) CometScan [native_comet] parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -190,7 +190,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -(18) CometScan [native_comet] parquet spark_catalog.default.store +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#16, s_store_id#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -218,7 +218,7 @@ Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(24) CometScan [native_comet] parquet spark_catalog.default.item +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_current_price#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -246,7 +246,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(30) CometScan [native_comet] parquet spark_catalog.default.promotion +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [2]: [p_promo_sk#21, p_channel_tv#22] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -298,7 +298,7 @@ Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_r Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#33, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#34, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#35] Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#33,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#34 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#35 AS profit#38, store channel AS channel#39, concat(store, s_store_id#18) AS id#40] -(41) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [7]: [cs_catalog_page_sk#41, cs_item_sk#42, cs_promo_sk#43, cs_order_number#44, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -318,7 +318,7 @@ Arguments: hashpartitioning(cs_item_sk#42, cs_order_number#44, 5), ENSURE_REQUIR Input [7]: [cs_catalog_page_sk#41, cs_item_sk#42, cs_promo_sk#43, cs_order_number#44, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47] Arguments: [cs_catalog_page_sk#41, cs_item_sk#42, cs_promo_sk#43, cs_order_number#44, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47], [cs_item_sk#42 ASC NULLS FIRST, cs_order_number#44 ASC NULLS FIRST] -(45) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#49, cr_order_number#50, cr_return_amount#51, cr_net_loss#52, cr_returned_date_sk#53] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -362,7 +362,7 @@ Arguments: [cs_sold_date_sk#47], [d_date_sk#54], Inner, BuildRight Input [9]: [cs_catalog_page_sk#41, cs_item_sk#42, cs_promo_sk#43, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47, cr_return_amount#51, cr_net_loss#52, d_date_sk#54] Arguments: [cs_catalog_page_sk#41, cs_item_sk#42, cs_promo_sk#43, cs_ext_sales_price#45, cs_net_profit#46, cr_return_amount#51, cr_net_loss#52], [cs_catalog_page_sk#41, cs_item_sk#42, cs_promo_sk#43, cs_ext_sales_price#45, cs_net_profit#46, cr_return_amount#51, cr_net_loss#52] -(55) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#55, cp_catalog_page_id#56] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -438,7 +438,7 @@ Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#45)), sum(coalesce(cast(cr_ Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#45))#70, sum(coalesce(cast(cr_return_amount#51 as decimal(12,2)), 0.00))#71, sum((cs_net_profit#46 - coalesce(cast(cr_net_loss#52 as decimal(12,2)), 0.00)))#72] Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#45))#70,17,2) AS sales#73, sum(coalesce(cast(cr_return_amount#51 as decimal(12,2)), 0.00))#71 AS returns#74, sum((cs_net_profit#46 - coalesce(cast(cr_net_loss#52 as decimal(12,2)), 0.00)))#72 AS profit#75, catalog channel AS channel#76, concat(catalog_page, cp_catalog_page_id#57) AS id#77] -(72) CometScan [native_comet] parquet spark_catalog.default.web_sales +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#78, ws_web_site_sk#79, ws_promo_sk#80, ws_order_number#81, ws_ext_sales_price#82, ws_net_profit#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -458,7 +458,7 @@ Arguments: hashpartitioning(ws_item_sk#78, ws_order_number#81, 5), ENSURE_REQUIR Input [7]: [ws_item_sk#78, ws_web_site_sk#79, ws_promo_sk#80, ws_order_number#81, ws_ext_sales_price#82, ws_net_profit#83, ws_sold_date_sk#84] Arguments: [ws_item_sk#78, ws_web_site_sk#79, ws_promo_sk#80, ws_order_number#81, ws_ext_sales_price#82, ws_net_profit#83, ws_sold_date_sk#84], [ws_item_sk#78 ASC NULLS FIRST, ws_order_number#81 ASC NULLS FIRST] -(76) CometScan [native_comet] parquet spark_catalog.default.web_returns +(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -502,7 +502,7 @@ Arguments: [ws_sold_date_sk#84], [d_date_sk#91], Inner, BuildRight Input [9]: [ws_item_sk#78, ws_web_site_sk#79, ws_promo_sk#80, ws_ext_sales_price#82, ws_net_profit#83, ws_sold_date_sk#84, wr_return_amt#88, wr_net_loss#89, d_date_sk#91] Arguments: [ws_item_sk#78, ws_web_site_sk#79, ws_promo_sk#80, ws_ext_sales_price#82, ws_net_profit#83, wr_return_amt#88, wr_net_loss#89], [ws_item_sk#78, ws_web_site_sk#79, ws_promo_sk#80, ws_ext_sales_price#82, ws_net_profit#83, wr_return_amt#88, wr_net_loss#89] -(86) CometScan [native_comet] parquet spark_catalog.default.web_site +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#92, web_site_id#93] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] @@ -616,10 +616,10 @@ BroadcastExchange (114) +- * CometColumnarToRow (113) +- CometProject (112) +- CometFilter (111) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (110) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (110) -(110) CometScan [native_comet] parquet spark_catalog.default.date_dim +(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt index 2d02e3f8a2..9a63144229 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt @@ -31,7 +31,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometExchange [ss_item_sk,ss_ticket_number] #3 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -39,28 +39,28 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] WholeStageCodegen (4) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow @@ -83,18 +83,18 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometExchange [cs_item_sk,cs_order_number] #11 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan [native_comet] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 WholeStageCodegen (6) @@ -119,17 +119,17 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometExchange [ws_item_sk,ws_order_number] #15 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt index e6643a3ac0..eb49bffa6a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt @@ -17,15 +17,15 @@ TakeOrderedAndProject (56) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : : +- BroadcastExchange (39) : : +- * Filter (38) : : +- * HashAggregate (37) @@ -42,22 +42,22 @@ TakeOrderedAndProject (56) : : :- CometProject (25) : : : +- CometBroadcastHashJoin (24) : : : :- CometFilter (22) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (21) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (21) : : : +- ReusedExchange (23) : : +- ReusedExchange (26) : +- BroadcastExchange (46) : +- * CometColumnarToRow (45) : +- CometProject (44) : +- CometFilter (43) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (42) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) +- BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (49) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -69,7 +69,7 @@ ReadSchema: struct (avg(ctr_total_ret Output [2]: [ctr_customer_sk#14, ctr_total_return#16] Input [5]: [ctr_customer_sk#14, ctr_state#15, ctr_total_return#16, (avg(ctr_total_return) * 1.2)#34, ctr_state#27] -(42) CometScan [native_comet] parquet spark_catalog.default.customer +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_sk#35, c_customer_id#36, c_current_addr_sk#37, c_salutation#38, c_first_name#39, c_last_name#40] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -284,7 +284,7 @@ Join condition: None Output [6]: [ctr_total_return#16, c_customer_id#41, c_current_addr_sk#37, c_salutation#42, c_first_name#43, c_last_name#44] Input [8]: [ctr_customer_sk#14, ctr_total_return#16, c_customer_sk#35, c_customer_id#41, c_current_addr_sk#37, c_salutation#42, c_first_name#43, c_last_name#44] -(49) CometScan [native_comet] parquet spark_catalog.default.customer_address +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [12]: [ca_address_sk#45, ca_street_number#46, ca_street_name#47, ca_street_type#48, ca_suite_number#49, ca_city#50, ca_county#51, ca_state#52, ca_zip#53, ca_country#54, ca_gmt_offset#55, ca_location_type#56] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -327,10 +327,10 @@ BroadcastExchange (61) +- * CometColumnarToRow (60) +- CometProject (59) +- CometFilter (58) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (57) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (57) -(57) CometScan [native_comet] parquet spark_catalog.default.date_dim +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt index dc46faa3df..66f68d490d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -28,15 +28,15 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) @@ -60,7 +60,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk,ca_state] #4 @@ -71,7 +71,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st InputAdapter CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] InputAdapter BroadcastExchange #9 WholeStageCodegen (6) @@ -79,4 +79,4 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st InputAdapter CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt index e1306d0d1c..d61d464b98 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt @@ -13,21 +13,21 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) +- CometProject (19) +- CometFilter (18) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (17) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -42,7 +42,7 @@ Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) A Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#2, 16)) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan [native_comet] parquet spark_catalog.default.inventory +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +71,7 @@ Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan [native_comet] parquet spark_catalog.default.store_sales +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#13, ss_sold_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt index 40fc82d84d..28d160a4a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt @@ -14,11 +14,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -26,11 +26,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [ss_item_sk] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt index 9274f49aa0..98e2738204 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt @@ -14,16 +14,16 @@ TakeOrderedAndProject (54) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- BroadcastExchange (35) : +- * HashAggregate (34) @@ -36,7 +36,7 @@ TakeOrderedAndProject (54) : :- CometProject (26) : : +- CometBroadcastHashJoin (25) : : :- CometFilter (23) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (22) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (22) : : +- ReusedExchange (24) : +- ReusedExchange (27) +- BroadcastExchange (51) @@ -50,12 +50,12 @@ TakeOrderedAndProject (54) :- CometProject (42) : +- CometBroadcastHashJoin (41) : :- CometFilter (39) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (38) : +- ReusedExchange (40) +- ReusedExchange (43) -(1) CometScan [native_comet] parquet spark_catalog.default.store_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -67,7 +67,7 @@ ReadSchema: struct Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -95,7 +95,7 @@ Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(sr_return_quantity#2)] Aggregate Attributes [1]: [sum(sr_return_quantity#2)#13] Results [2]: [i_item_id#7 AS item_id#14, sum(sr_return_quantity#2)#13 AS sr_item_qty#15] -(22) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#16, cr_return_quantity#17, cr_returned_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -229,7 +229,7 @@ Join condition: None Output [3]: [item_id#14, sr_item_qty#15, cr_item_qty#27] Input [4]: [item_id#14, sr_item_qty#15, item_id#26, cr_item_qty#27] -(38) CometScan [native_comet] parquet spark_catalog.default.web_returns +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] Batched: true Location: InMemoryFileIndex [] @@ -315,18 +315,18 @@ BroadcastExchange (68) +- CometProject (66) +- CometBroadcastHashJoin (65) :- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (64) +- CometProject (63) +- CometBroadcastHashJoin (62) - :- CometScan [native_comet] parquet spark_catalog.default.date_dim (57) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (57) +- CometBroadcastExchange (61) +- CometProject (60) +- CometFilter (59) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (58) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -337,13 +337,13 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(57) CometScan [native_comet] parquet spark_catalog.default.date_dim +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#10, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(58) CometScan [native_comet] parquet spark_catalog.default.date_dim +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#45, d_week_seq#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt index 98e6aeefe7..8cdd84fda4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt @@ -17,7 +17,7 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -26,24 +26,24 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 InputAdapter BroadcastExchange #7 @@ -61,7 +61,7 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #5 ReusedExchange [d_date_sk] #6 @@ -81,7 +81,7 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #5 ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt index 5b156893cf..c340659028 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/explain.txt @@ -14,27 +14,27 @@ TakeOrderedAndProject (33) : : : : +- CometBroadcastHashJoin (8) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) : : : +- CometBroadcastExchange (12) : : : +- CometFilter (11) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.income_band (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.store_returns (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -49,7 +49,7 @@ Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2) Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#1, 16)) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#5, 20)) AS c_first_name#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#6, 30)) AS c_last_name#9] -(4) CometScan [native_comet] parquet spark_catalog.default.customer_address +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#10, ca_city#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -77,7 +77,7 @@ Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(10) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -101,7 +101,7 @@ Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(15) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -125,7 +125,7 @@ Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(20) CometScan [native_comet] parquet spark_catalog.default.income_band +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -157,7 +157,7 @@ Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_c Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(27) CometScan [native_comet] parquet spark_catalog.default.store_returns +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt index 6936907ec8..cc7cf901e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84/simplified.txt @@ -15,21 +15,21 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometScan [native_comet] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt index 07098c43dc..9fa8767b11 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt @@ -21,36 +21,36 @@ TakeOrderedAndProject (49) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometBroadcastExchange (3) : : : : : : : +- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_page (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometProject (16) : : : : +- CometFilter (15) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) +- CometBroadcastExchange (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_comet] parquet spark_catalog.default.reason (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -66,7 +66,7 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometScan [native_comet] parquet spark_catalog.default.web_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -90,7 +90,7 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(9) CometScan [native_comet] parquet spark_catalog.default.web_page +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -114,7 +114,7 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(14) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -142,7 +142,7 @@ Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_s Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -170,7 +170,7 @@ Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status# Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(26) CometScan [native_comet] parquet spark_catalog.default.customer_address +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -198,7 +198,7 @@ Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(32) CometScan [native_comet] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -226,7 +226,7 @@ Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(38) CometScan [native_comet] parquet spark_catalog.default.reason +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#35, r_reason_desc#36] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -289,10 +289,10 @@ BroadcastExchange (54) +- * CometColumnarToRow (53) +- CometProject (52) +- CometFilter (51) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (50) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) -(50) CometScan [native_comet] parquet spark_catalog.default.date_dim +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt index 4417a5dceb..3d6c0f9695 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt @@ -24,7 +24,7 @@ TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refu CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -32,30 +32,30 @@ TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refu InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometBroadcastExchange [wp_web_page_sk] #4 CometFilter [wp_web_page_sk] - CometScan [native_comet] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #7 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt index b5c0aebb42..d4343c33be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt @@ -16,18 +16,18 @@ TakeOrderedAndProject (26) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -67,7 +67,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -153,10 +153,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt index e220ef060c..c3f8e3a26c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt @@ -23,7 +23,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -31,12 +31,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt index 78906678a2..3dd56d08f4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt @@ -15,15 +15,15 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : +- BroadcastExchange (31) : +- * CometColumnarToRow (30) : +- CometHashAggregate (29) @@ -34,7 +34,7 @@ : :- CometProject (23) : : +- CometBroadcastHashJoin (22) : : :- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (19) : : +- ReusedExchange (21) : +- ReusedExchange (24) +- BroadcastExchange (45) @@ -47,12 +47,12 @@ :- CometProject (37) : +- CometBroadcastHashJoin (36) : :- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : +- ReusedExchange (35) +- ReusedExchange (38) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -92,7 +92,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_comet] parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -137,7 +137,7 @@ Functions: [] (18) CometColumnarToRow [codegen id : 3] Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -(19) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -200,7 +200,7 @@ Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_ Join type: LeftAnti Join condition: None -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -295,10 +295,10 @@ BroadcastExchange (56) +- * CometColumnarToRow (55) +- CometProject (54) +- CometFilter (53) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (52) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) -(52) CometScan [native_comet] parquet spark_catalog.default.date_dim +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt index 6b09d49e07..f687139735 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt @@ -18,7 +18,7 @@ WholeStageCodegen (4) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -26,15 +26,15 @@ WholeStageCodegen (4) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (1) @@ -48,7 +48,7 @@ WholeStageCodegen (4) CometProject [cs_bill_customer_sk,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 @@ -65,7 +65,7 @@ WholeStageCodegen (4) CometProject [ws_bill_customer_sk,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt index 6ff7198461..2e49d20af4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt @@ -18,19 +18,19 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) +: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (16) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) : : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) @@ -44,12 +44,12 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (26) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (32) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) : : : : : +- * CometColumnarToRow (65) @@ -64,12 +64,12 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (47) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (53) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) : : : : +- * CometColumnarToRow (86) @@ -84,12 +84,12 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (68) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (74) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) : : : +- * CometColumnarToRow (107) @@ -104,12 +104,12 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (89) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (95) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) : : +- * CometColumnarToRow (128) @@ -124,12 +124,12 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (110) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (116) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) : +- * CometColumnarToRow (149) @@ -144,12 +144,12 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (131) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (137) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) +- * CometColumnarToRow (170) @@ -164,16 +164,16 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (152) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (158) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -188,7 +188,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -216,7 +216,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -244,7 +244,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -288,7 +288,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometScan [native_comet] parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -315,7 +315,7 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometScan [native_comet] parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -379,7 +379,7 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometScan [native_comet] parquet spark_catalog.default.store_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -406,7 +406,7 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometScan [native_comet] parquet spark_catalog.default.time_dim +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -470,7 +470,7 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometScan [native_comet] parquet spark_catalog.default.store_sales +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -497,7 +497,7 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometScan [native_comet] parquet spark_catalog.default.time_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -561,7 +561,7 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometScan [native_comet] parquet spark_catalog.default.store_sales +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -588,7 +588,7 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometScan [native_comet] parquet spark_catalog.default.time_dim +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -652,7 +652,7 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometScan [native_comet] parquet spark_catalog.default.store_sales +(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -679,7 +679,7 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometScan [native_comet] parquet spark_catalog.default.time_dim +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -743,7 +743,7 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometScan [native_comet] parquet spark_catalog.default.store_sales +(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -770,7 +770,7 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometScan [native_comet] parquet spark_catalog.default.time_dim +(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -834,7 +834,7 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometScan [native_comet] parquet spark_catalog.default.store_sales +(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -861,7 +861,7 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometScan [native_comet] parquet spark_catalog.default.time_dim +(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt index c47266d366..3490714189 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt @@ -19,19 +19,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -48,12 +48,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 @@ -71,12 +71,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 @@ -94,12 +94,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 @@ -117,12 +117,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 @@ -140,12 +140,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 @@ -163,12 +163,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt index 80ce3b05da..a6b45de2e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt @@ -19,20 +19,20 @@ TakeOrderedAndProject (31) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -47,7 +47,7 @@ Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category# Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) AS i_class#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) AS i_category#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -72,7 +72,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -100,7 +100,7 @@ Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -182,10 +182,10 @@ BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(32) CometScan [native_comet] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt index b3a2755b66..95eb8a0a79 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt @@ -26,10 +26,10 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] CometFilter [i_item_sk,i_brand,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -37,11 +37,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt index 6f5428bc26..4d3bc8a556 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt @@ -2,10 +2,10 @@ * Project (4) +- * CometColumnarToRow (3) +- CometFilter (2) - +- CometScan [native_comet] parquet spark_catalog.default.reason (1) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) -(1) CometScan [native_comet] parquet spark_catalog.default.reason +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -34,10 +34,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- * CometColumnarToRow (8) +- CometProject (7) +- CometFilter (6) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) -(5) CometScan [native_comet] parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -93,10 +93,10 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- * CometColumnarToRow (17) +- CometProject (16) +- CometFilter (15) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (14) -(14) CometScan [native_comet] parquet spark_catalog.default.store_sales +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -152,10 +152,10 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- * CometColumnarToRow (26) +- CometProject (25) +- CometFilter (24) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) -(23) CometScan [native_comet] parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -211,10 +211,10 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (32) -(32) CometScan [native_comet] parquet spark_catalog.default.store_sales +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -270,10 +270,10 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (41) -(41) CometScan [native_comet] parquet spark_catalog.default.store_sales +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt index dac4ebf393..daebc7bff8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) InputAdapter CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 @@ -29,7 +29,7 @@ WholeStageCodegen (1) InputAdapter CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 @@ -45,7 +45,7 @@ WholeStageCodegen (1) InputAdapter CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 @@ -61,7 +61,7 @@ WholeStageCodegen (1) InputAdapter CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 @@ -77,10 +77,10 @@ WholeStageCodegen (1) InputAdapter CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 CometColumnarToRow InputAdapter CometFilter [r_reason_sk] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt index b3916111e7..08ae744b5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt @@ -13,19 +13,19 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometScan [native_comet] parquet spark_catalog.default.web_page (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometHashAggregate (43) @@ -39,16 +39,16 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -63,7 +63,7 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -91,7 +91,7 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -119,7 +119,7 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.web_page +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -163,7 +163,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometScan [native_comet] parquet spark_catalog.default.web_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -190,7 +190,7 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometScan [native_comet] parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt index 1a5ae5b626..2db3b6c957 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometScan [native_comet] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt index 1e90ccb788..ab12f9e8b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt @@ -21,32 +21,32 @@ : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometProject (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometFilter (5) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) : : +- CometBroadcastExchange (23) : : +- CometProject (22) : : +- CometFilter (21) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) -(1) CometScan [native_comet] parquet spark_catalog.default.call_center +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -61,7 +61,7 @@ Condition : isnotnull(cc_call_center_sk#1) Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(cc_call_center_id#2, 16)) AS cc_call_center_id#5, cc_name#3, cc_manager#4] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -86,7 +86,7 @@ Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -114,7 +114,7 @@ Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(15) CometScan [native_comet] parquet spark_catalog.default.customer +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -138,7 +138,7 @@ Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_address +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -166,7 +166,7 @@ Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -(26) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -194,7 +194,7 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -(32) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#25, hd_buy_potential#26] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -264,10 +264,10 @@ BroadcastExchange (50) +- * CometColumnarToRow (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_comet] parquet spark_catalog.default.date_dim +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt index ddf001af79..23da984036 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt @@ -26,10 +26,10 @@ WholeStageCodegen (3) CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -37,23 +37,23 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #7 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk] #9 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt index 79b5eab9ab..3cd61cf1a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt @@ -11,11 +11,11 @@ : : +- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- BroadcastExchange (24) : +- * Filter (23) : +- * HashAggregate (22) @@ -26,15 +26,15 @@ : +- CometProject (17) : +- CometBroadcastHashJoin (16) : :- CometFilter (11) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (10) : +- CometBroadcastExchange (15) : +- CometProject (14) : +- CometFilter (13) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (12) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) +- ReusedExchange (27) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -77,7 +77,7 @@ Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5], [ws_ext_disc (9) CometColumnarToRow [codegen id : 4] Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -(10) CometScan [native_comet] parquet spark_catalog.default.web_sales +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -89,7 +89,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Condition : isnotnull(ws_item_sk#7) -(12) CometScan [native_comet] parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -200,10 +200,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt index 82414bda15..35949ed2e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt @@ -14,7 +14,7 @@ WholeStageCodegen (5) CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -22,11 +22,11 @@ WholeStageCodegen (5) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) @@ -42,11 +42,11 @@ WholeStageCodegen (5) CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt index 83febb90bd..320f50c7e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt @@ -12,19 +12,19 @@ TakeOrderedAndProject (23) : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometScan [native_comet] parquet spark_catalog.default.reason (12) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -42,7 +42,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -74,7 +74,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometScan [native_comet] parquet spark_catalog.default.reason +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt index b4af2ce3b3..0335e86808 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/simplified.txt @@ -15,13 +15,13 @@ TakeOrderedAndProject [sumsales,ss_customer_sk] CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometExchange [ss_item_sk,ss_ticket_number] #2 CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometExchange [sr_item_sk,sr_ticket_number] #3 CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [r_reason_sk] #4 CometProject [r_reason_sk] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt index 60e794db95..97b00ebbbd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (12) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (23) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -92,7 +92,7 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometScan [native_comet] parquet spark_catalog.default.web_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -115,7 +115,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometScan [native_comet] parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometScan [native_comet] parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometScan [native_comet] parquet spark_catalog.default.web_site +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt index 9ca9bdf113..f2b4e7f1bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [wr_order_number] CometExchange [wr_order_number] #4 CometProject [wr_order_number] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt index 075ebfabac..205195f2ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt @@ -18,14 +18,14 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometProject (14) : : : : +- CometSortMergeJoin (13) : : : : :- CometSort (10) : : : : : +- CometExchange (9) : : : : : +- CometProject (8) : : : : : +- CometFilter (7) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) : : : : +- CometSort (12) : : : : +- ReusedExchange (11) : : : +- CometProject (28) @@ -34,7 +34,7 @@ : : : : +- CometExchange (19) : : : : +- CometProject (18) : : : : +- CometFilter (17) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (16) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) : : : +- CometProject (26) : : : +- CometSortMergeJoin (25) : : : :- CometSort (22) @@ -44,18 +44,18 @@ : : +- CometBroadcastExchange (33) : : +- CometProject (32) : : +- CometFilter (31) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : +- CometBroadcastExchange (39) : +- CometProject (38) : +- CometFilter (37) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (36) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) +- CometBroadcastExchange (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -78,7 +78,7 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -122,7 +122,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometScan [native_comet] parquet spark_catalog.default.web_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -182,7 +182,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -210,7 +210,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_address +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#21, ca_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -238,7 +238,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometScan [native_comet] parquet spark_catalog.default.web_site +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#23, web_company_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt index 81f21ceb72..3a9645f4fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt @@ -21,14 +21,14 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 CometProject [wr_order_number] @@ -37,7 +37,7 @@ WholeStageCodegen (2) CometExchange [wr_order_number] #4 CometProject [wr_order_number] CometFilter [wr_order_number,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] @@ -47,12 +47,12 @@ WholeStageCodegen (2) CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt index 86b62b1982..f90d8b12b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt @@ -11,22 +11,22 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometScan [native_comet] parquet spark_catalog.default.store (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -69,7 +69,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -97,7 +97,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt index 34e10d3a24..36f4009885 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt index 2c88daabfe..1f1b4bb23e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt @@ -12,29 +12,29 @@ : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (2) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) - :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (12) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) +- ReusedExchange (13) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] ReadSchema: struct -(2) CometScan [native_comet] parquet spark_catalog.default.date_dim +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -80,7 +80,7 @@ Functions: [] Input [2]: [customer_sk#7, item_sk#8] Arguments: [customer_sk#7, item_sk#8], [customer_sk#7 ASC NULLS FIRST, item_sk#8 ASC NULLS FIRST] -(12) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -157,10 +157,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt index b6f1ea418b..5c52e9d58a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (2) CometHashAggregate [ss_customer_sk,ss_item_sk] CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -23,17 +23,17 @@ WholeStageCodegen (2) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] CometExchange [cs_bill_customer_sk,cs_item_sk] #5 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt index 854e78a0a3..a7208612a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt @@ -18,18 +18,18 @@ :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -41,7 +41,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -69,7 +69,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -162,10 +162,10 @@ BroadcastExchange (33) +- * CometColumnarToRow (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (29) -(29) CometScan [native_comet] parquet spark_catalog.default.date_dim +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt index c153bbdc7f..74db311e7b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt @@ -27,7 +27,7 @@ WholeStageCodegen (5) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -35,12 +35,12 @@ WholeStageCodegen (5) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt index 90ed9ef2b5..8e1f1495d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt @@ -14,24 +14,24 @@ TakeOrderedAndProject (30) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.ship_mode (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) : +- CometBroadcastExchange (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) +- CometBroadcastExchange (22) +- CometProject (21) +- CometFilter (20) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -42,7 +42,7 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#4] Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt index dcab21d624..e7193f87e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt index 8dd34e58ed..091ad62567 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt @@ -17,13 +17,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -34,13 +34,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (52) : +- CometFilter (51) : +- CometHashAggregate (50) @@ -52,10 +52,10 @@ : : +- CometBroadcastHashJoin (43) : : :- CometProject (39) : : : +- CometFilter (38) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) : : +- CometBroadcastExchange (42) : : +- CometFilter (41) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : +- ReusedExchange (45) +- CometBroadcastExchange (69) +- CometHashAggregate (68) @@ -67,14 +67,14 @@ : +- CometBroadcastHashJoin (61) : :- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (55) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) : +- CometBroadcastExchange (60) : +- CometFilter (59) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (58) +- ReusedExchange (63) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -89,7 +89,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -114,7 +114,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -156,7 +156,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -171,7 +171,7 @@ Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Batched: true Location: InMemoryFileIndex [] @@ -196,7 +196,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -247,7 +247,7 @@ Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight Input [5]: [customer_id#23, year_total#24, customer_id#41, customer_preferred_cust_flag#42, year_total#43] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] -(37) CometScan [native_comet] parquet spark_catalog.default.customer +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -262,7 +262,7 @@ Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] -(40) CometScan [native_comet] parquet spark_catalog.default.web_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] Batched: true Location: InMemoryFileIndex [] @@ -330,7 +330,7 @@ Arguments: [customer_id#23], [customer_id#66], Inner, BuildRight Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, customer_id#66, year_total#67] Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] -(55) CometScan [native_comet] parquet spark_catalog.default.customer +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -345,7 +345,7 @@ Condition : (isnotnull(c_customer_sk#68) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] Arguments: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57], [c_customer_sk#68, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#69, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#70, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#71, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#72, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#73, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#74, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#75, 50, true, false, true) AS c_email_address#57] -(58) CometScan [native_comet] parquet spark_catalog.default.web_sales +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] @@ -422,10 +422,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometFilter (75) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) -(74) CometScan [native_comet] parquet spark_catalog.default.date_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -447,10 +447,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometFilter (79) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (78) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (78) -(78) CometScan [native_comet] parquet spark_catalog.default.date_dim +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt index 131ebf2656..a3bacdf562 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt @@ -18,20 +18,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 CometHashAggregate [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -42,20 +42,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -67,10 +67,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -83,9 +83,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt index 88341ba737..93308784a9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt @@ -13,18 +13,18 @@ TakeOrderedAndProject (23) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt index f48019d841..0a1a7d8020 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt index 9b7dbcb6e6..74da8ba883 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt @@ -14,28 +14,28 @@ : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) +- CometBroadcastExchange (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -47,7 +47,7 @@ ReadSchema: struct= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometScan [native_comet] parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -71,7 +71,7 @@ Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -99,7 +99,7 @@ Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -155,7 +155,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -(26) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#24, hd_dep_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -203,10 +203,10 @@ BroadcastExchange (39) +- * CometColumnarToRow (38) +- CometProject (37) +- CometFilter (36) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) -(35) CometScan [native_comet] parquet spark_catalog.default.date_dim +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#17, d_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt index 2511aa3485..a1a0200121 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -23,22 +23,22 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt index 220e7e48fa..0289abc42d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt @@ -17,12 +17,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -33,25 +33,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -59,18 +59,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) :- CometProject (80) : +- CometFilter (79) : +- CometHashAggregate (78) @@ -82,7 +82,7 @@ : : +- CometBroadcastHashJoin (71) : : :- CometBroadcastHashJoin (69) : : : :- CometFilter (67) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (66) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (66) : : : +- ReusedExchange (68) : : +- ReusedExchange (70) : +- ReusedExchange (73) @@ -97,13 +97,13 @@ : +- CometBroadcastHashJoin (86) : :- CometBroadcastHashJoin (84) : : :- CometFilter (82) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (81) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (81) : : +- ReusedExchange (83) : +- ReusedExchange (85) +- ReusedExchange (88) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -126,7 +126,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -149,7 +149,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -185,7 +185,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -261,7 +261,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -328,7 +328,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -360,7 +360,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,7 +410,7 @@ Condition : (isnotnull(sales#49) AND (cast(sales#49 as decimal(32,6)) > cast(Sub Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] Arguments: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56], [sales#49, number_sales#50, store AS channel#53, i_brand_id#40 AS i_brand_id#54, i_class_id#41 AS i_class_id#55, i_category_id#42 AS i_category_id#56] -(66) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] @@ -476,7 +476,7 @@ Condition : (isnotnull(sales#70) AND (cast(sales#70 as decimal(32,6)) > cast(Reu Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#70, number_sales#71] Arguments: [sales#70, number_sales#71, channel#72, i_brand_id#63, i_class_id#64, i_category_id#65], [sales#70, number_sales#71, catalog AS channel#72, i_brand_id#63, i_class_id#64, i_category_id#65] -(81) CometScan [native_comet] parquet spark_catalog.default.web_sales +(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#73, ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76] Batched: true Location: InMemoryFileIndex [] @@ -582,19 +582,19 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (115) :- CometProject (106) : +- CometBroadcastHashJoin (105) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (103) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (103) : +- ReusedExchange (104) :- CometProject (110) : +- CometBroadcastHashJoin (109) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (107) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (107) : +- ReusedExchange (108) +- CometProject (114) +- CometBroadcastHashJoin (113) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (111) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (111) +- ReusedExchange (112) -(103) CometScan [native_comet] parquet spark_catalog.default.store_sales +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#99, ss_list_price#100, ss_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] @@ -613,7 +613,7 @@ Arguments: [ss_sold_date_sk#101], [d_date_sk#103], Inner, BuildRight Input [4]: [ss_quantity#99, ss_list_price#100, ss_sold_date_sk#101, d_date_sk#103] Arguments: [quantity#104, list_price#105], [ss_quantity#99 AS quantity#104, ss_list_price#100 AS list_price#105] -(107) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#106, cs_list_price#107, cs_sold_date_sk#108] Batched: true Location: InMemoryFileIndex [] @@ -632,7 +632,7 @@ Arguments: [cs_sold_date_sk#108], [d_date_sk#110], Inner, BuildRight Input [4]: [cs_quantity#106, cs_list_price#107, cs_sold_date_sk#108, d_date_sk#110] Arguments: [quantity#111, list_price#112], [cs_quantity#106 AS quantity#111, cs_list_price#107 AS list_price#112] -(111) CometScan [native_comet] parquet spark_catalog.default.web_sales +(111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115] Batched: true Location: InMemoryFileIndex [] @@ -684,10 +684,10 @@ BroadcastExchange (124) +- * CometColumnarToRow (123) +- CometProject (122) +- CometFilter (121) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (120) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (120) -(120) CometScan [native_comet] parquet spark_catalog.default.date_dim +(120) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -714,10 +714,10 @@ BroadcastExchange (129) +- * CometColumnarToRow (128) +- CometProject (127) +- CometFilter (126) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (125) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) -(125) CometScan [native_comet] parquet spark_catalog.default.date_dim +(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt index 730f748777..13c79b4299 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt @@ -19,17 +19,17 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -49,12 +49,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -65,7 +65,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #7 WholeStageCodegen (1) @@ -73,26 +73,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 CometProject [i_brand_id,i_class_id,i_category_id] @@ -100,19 +100,19 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 ReusedExchange [d_date_sk] #11 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #4 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 @@ -125,7 +125,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 @@ -142,7 +142,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt index d7483bdd31..bcef4db073 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt @@ -12,12 +12,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -28,25 +28,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -54,18 +54,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -77,16 +77,16 @@ : +- CometBroadcastHashJoin (70) : :- CometBroadcastHashJoin (68) : : :- CometFilter (66) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) : : +- ReusedExchange (67) : +- ReusedExchange (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -109,7 +109,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -132,7 +132,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -244,7 +244,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -389,7 +389,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) -(65) CometScan [native_comet] parquet spark_catalog.default.store_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#60], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] -(72) CometScan [native_comet] parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -493,19 +493,19 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (98) :- CometProject (89) : +- CometBroadcastHashJoin (88) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (86) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) : +- ReusedExchange (87) :- CometProject (93) : +- CometBroadcastHashJoin (92) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (90) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) : +- ReusedExchange (91) +- CometProject (97) +- CometBroadcastHashJoin (96) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (94) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) +- ReusedExchange (95) -(86) CometScan [native_comet] parquet spark_catalog.default.store_sales +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76] Batched: true Location: InMemoryFileIndex [] @@ -524,7 +524,7 @@ Arguments: [ss_sold_date_sk#76], [d_date_sk#78], Inner, BuildRight Input [4]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76, d_date_sk#78] Arguments: [quantity#79, list_price#80], [ss_quantity#74 AS quantity#79, ss_list_price#75 AS list_price#80] -(90) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] @@ -543,7 +543,7 @@ Arguments: [cs_sold_date_sk#83], [d_date_sk#85], Inner, BuildRight Input [4]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83, d_date_sk#85] Arguments: [quantity#86, list_price#87], [cs_quantity#81 AS quantity#86, cs_list_price#82 AS list_price#87] -(94) CometScan [native_comet] parquet spark_catalog.default.web_sales +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90] Batched: true Location: InMemoryFileIndex [] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (103) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_comet] parquet spark_catalog.default.date_dim +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_comet] parquet spark_catalog.default.date_dim +(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#99] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (117) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_comet] parquet spark_catalog.default.date_dim +(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (122) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_comet] parquet spark_catalog.default.date_dim +(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt index 0e0b2ea4b8..5d5557bc59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) @@ -75,26 +75,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -102,20 +102,20 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -128,7 +128,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #17 WholeStageCodegen (1) @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt index d6b99398ca..af9b2efbd1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt @@ -11,21 +11,21 @@ : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) +- CometBroadcastExchange (17) +- CometProject (16) +- CometFilter (15) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.customer +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#5, c_current_addr_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -61,7 +61,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -89,7 +89,7 @@ Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_comet] parquet spark_catalog.default.date_dim +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt index 6105b85eb5..6eac0d3bd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt index a7a73163fd..1654e70a54 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (12) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (23) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.call_center (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -92,7 +92,7 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -115,7 +115,7 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometScan [native_comet] parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometScan [native_comet] parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometScan [native_comet] parquet spark_catalog.default.call_center +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt index 96c978299d..352d9e22b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [cs_order_number] #2 CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] CometSort [cs_warehouse_sk,cs_order_number] CometExchange [cs_order_number] #3 CometProject [cs_warehouse_sk,cs_order_number] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] CometSort [cr_order_number] CometExchange [cr_order_number] #4 CometProject [cr_order_number] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cc_call_center_sk] #7 CometProject [cc_call_center_sk] CometFilter [cc_call_center_sk,cc_county] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt index d37f4a2a69..f502a5cd8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt @@ -19,33 +19,33 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_comet] parquet spark_catalog.default.store (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.item (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -57,7 +57,7 @@ ReadSchema: struct Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometScan [native_comet] parquet spark_catalog.default.date_dim +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#1 Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(15) CometScan [native_comet] parquet spark_catalog.default.date_dim +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -139,7 +139,7 @@ Input [8]: [d_week_seq#10, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#4 Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(23) CometScan [native_comet] parquet spark_catalog.default.date_dim +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt index 9d342c2218..25ed325468 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] CometUnion [sold_date_sk,sales_price] CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] @@ -33,4 +33,4 @@ WholeStageCodegen (1) CometBroadcastExchange [d_week_seq] #6 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt index 51fba80c47..c23383e09a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt @@ -13,18 +13,18 @@ TakeOrderedAndProject (23) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt index 67283a9aba..fdf7f7ffa6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt index 83d5f82979..5656916223 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt @@ -12,20 +12,20 @@ : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan [native_comet] parquet spark_catalog.default.item (8) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +37,7 @@ ReadSchema: struct -(2) CometScan [native_comet] parquet spark_catalog.default.store_sales +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -87,7 +87,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -115,7 +115,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(10) CometScan [native_comet] parquet spark_catalog.default.item +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -182,7 +182,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(25) CometScan [native_comet] parquet spark_catalog.default.store_sales +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -197,7 +197,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(28) CometScan [native_comet] parquet spark_catalog.default.customer +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,7 +256,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -284,7 +284,7 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#29], Inner, BuildRight Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#29] Arguments: [sales#32], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#32] -(47) CometScan [native_comet] parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] Batched: true Location: InMemoryFileIndex [] @@ -380,10 +380,10 @@ BroadcastExchange (72) +- * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_comet] parquet spark_catalog.default.date_dim +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,10 +410,10 @@ BroadcastExchange (77) +- * CometColumnarToRow (76) +- CometProject (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -448,15 +448,15 @@ Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquer :- CometProject (82) : +- CometBroadcastHashJoin (81) : :- CometFilter (79) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (78) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (78) : +- ReusedExchange (80) +- CometBroadcastExchange (86) +- CometProject (85) +- CometFilter (84) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (83) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (83) -(78) CometScan [native_comet] parquet spark_catalog.default.store_sales +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#44, ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -480,7 +480,7 @@ Arguments: [ss_customer_sk#44], [c_customer_sk#49], Inner, BuildRight Input [5]: [ss_customer_sk#44, ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49] Arguments: [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49], [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49] -(83) CometScan [native_comet] parquet spark_catalog.default.date_dim +(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#50, d_year#51] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -544,10 +544,10 @@ BroadcastExchange (100) +- * CometColumnarToRow (99) +- CometProject (98) +- CometFilter (97) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (96) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (96) -(96) CometScan [native_comet] parquet spark_catalog.default.date_dim +(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#50, d_year#51] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt index 7fed1a3233..cd74cced17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -21,7 +21,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -33,7 +33,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) @@ -41,14 +41,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #13 WholeStageCodegen (1) @@ -75,12 +75,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] @@ -88,14 +88,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_quantity,ws_list_price] [sales] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] @@ -104,7 +104,7 @@ WholeStageCodegen (1) CometExchange [ws_bill_customer_sk] #16 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [item_sk] #4 CometSort [c_customer_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt index e4f754c01a..67e4e39057 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt @@ -15,7 +15,7 @@ : : : : +- CometProject (23) : : : : +- CometBroadcastHashJoin (22) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (21) : : : : +- CometProject (20) : : : : +- CometFilter (19) @@ -27,14 +27,14 @@ : : : : :- CometProject (10) : : : : : +- CometBroadcastHashJoin (9) : : : : : :- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : : +- CometBroadcastExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) : : : : +- CometBroadcastExchange (13) : : : : +- CometFilter (12) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : +- CometSort (39) : : : +- CometProject (38) : : : +- CometFilter (37) @@ -45,17 +45,17 @@ : : : +- CometBroadcastHashJoin (32) : : : :- CometProject (28) : : : : +- CometFilter (27) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (26) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) : : : +- CometBroadcastExchange (31) : : : +- CometFilter (30) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (29) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) : : +- CometBroadcastExchange (52) : : +- CometProject (51) : : +- CometSortMergeJoin (50) : : :- CometSort (44) : : : +- CometExchange (43) : : : +- CometFilter (42) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (41) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) : : +- CometSort (49) : : +- CometProject (48) : : +- CometFilter (47) @@ -64,7 +64,7 @@ : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometHashAggregate (85) +- CometExchange (84) +- CometHashAggregate (83) @@ -78,7 +78,7 @@ : : : +- CometProject (68) : : : +- CometBroadcastHashJoin (67) : : : :- CometFilter (65) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (64) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (64) : : : +- ReusedExchange (66) : : +- CometSort (75) : : +- CometProject (74) @@ -89,7 +89,7 @@ +- ReusedExchange (80) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -101,7 +101,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] Condition : isnotnull(ss_item_sk#7) -(5) CometScan [native_comet] parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -141,7 +141,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#13, i_item_desc#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -208,7 +208,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(26) CometScan [native_comet] parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -223,7 +223,7 @@ Condition : isnotnull(ss_customer_sk#19) Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -(29) CometScan [native_comet] parquet spark_catalog.default.customer +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -278,7 +278,7 @@ Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold Right output [1]: [c_customer_sk#23] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi -(41) CometScan [native_comet] parquet spark_catalog.default.customer +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -339,7 +339,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#29], Inner, BuildRight Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#29, c_first_name#32, c_last_name#33] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -381,7 +381,7 @@ Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] Keys [2]: [c_last_name#33, c_first_name#32] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(64) CometScan [native_comet] parquet spark_catalog.default.web_sales +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -494,10 +494,10 @@ BroadcastExchange (93) +- * CometColumnarToRow (92) +- CometProject (91) +- CometFilter (90) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (89) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (89) -(89) CometScan [native_comet] parquet spark_catalog.default.date_dim +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#34, d_year#35, d_moy#36] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -524,10 +524,10 @@ BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometProject (96) +- CometFilter (95) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (94) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) -(94) CometScan [native_comet] parquet spark_catalog.default.date_dim +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -562,15 +562,15 @@ Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquer :- CometProject (103) : +- CometBroadcastHashJoin (102) : :- CometFilter (100) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (99) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (99) : +- ReusedExchange (101) +- CometBroadcastExchange (107) +- CometProject (106) +- CometFilter (105) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) -(99) CometScan [native_comet] parquet spark_catalog.default.store_sales +(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#53, ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] @@ -594,7 +594,7 @@ Arguments: [ss_customer_sk#53], [c_customer_sk#58], Inner, BuildRight Input [5]: [ss_customer_sk#53, ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58] Arguments: [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58], [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58] -(104) CometScan [native_comet] parquet spark_catalog.default.date_dim +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#59, d_year#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -658,10 +658,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (117) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_comet] parquet spark_catalog.default.date_dim +(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#59, d_year#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt index 96a3fd3d50..0575c4a672 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,7 +24,7 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [item_sk] #4 CometProject [item_sk] CometFilter [item_sk,cnt] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #6 WholeStageCodegen (1) @@ -44,14 +44,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #8 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -70,7 +70,7 @@ WholeStageCodegen (1) CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #13 WholeStageCodegen (1) @@ -78,12 +78,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [c_customer_sk] #10 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] @@ -91,17 +91,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #10 CometFilter [c_customer_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] CometExchange [c_customer_sk] #16 CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] @@ -111,7 +111,7 @@ WholeStageCodegen (1) CometBroadcastExchange [d_date_sk] #17 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] CometExchange [c_last_name,c_first_name] #18 CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,ws_quantity,ws_list_price] @@ -125,7 +125,7 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [item_sk] #4 CometSort [c_customer_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt index fb57800f44..e478cdfc01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_comet] parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_comet] parquet spark_catalog.default.item +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_comet] parquet spark_catalog.default.customer_address +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_comet] parquet spark_catalog.default.item +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt index 8444341b4c..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt index 612595b71a..548959be1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt @@ -23,32 +23,32 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometSort (10) : : : : +- CometExchange (9) : : : : +- CometProject (8) : : : : +- CometFilter (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) : : : +- CometBroadcastExchange (16) : : : +- CometProject (15) : : : +- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) : : +- CometBroadcastExchange (22) : : +- CometProject (21) : : +- CometFilter (20) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) : +- CometBroadcastExchange (28) : +- CometProject (27) : +- CometFilter (26) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) +- BroadcastExchange (36) +- * CometColumnarToRow (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -71,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -103,7 +103,7 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan [native_comet] parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -131,7 +131,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan [native_comet] parquet spark_catalog.default.item +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -190,7 +190,7 @@ Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_pric (31) CometColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) CometScan [native_comet] parquet spark_catalog.default.customer_address +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -298,7 +298,7 @@ Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquer : : +- CometBroadcastExchange (60) : : +- CometProject (59) : : +- CometFilter (58) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) : +- ReusedExchange (63) +- ReusedExchange (67) @@ -338,7 +338,7 @@ Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] -(57) CometScan [native_comet] parquet spark_catalog.default.item +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt index 8444341b4c..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt @@ -34,7 +34,7 @@ WholeStageCodegen (4) CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 InputAdapter ReusedExchange [ca_state,ca_zip,ca_country] #8 @@ -66,24 +66,24 @@ WholeStageCodegen (4) CometExchange [ss_ticket_number,ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #4 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) @@ -91,4 +91,4 @@ WholeStageCodegen (4) InputAdapter CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] CometFilter [ca_state,ca_zip,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt index 2b4aac2e83..6a9f9094d3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt @@ -19,33 +19,33 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan [native_comet] parquet spark_catalog.default.store (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) +- CometBroadcastExchange (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.item (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -57,7 +57,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_county#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -165,7 +165,7 @@ Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] Keys [3]: [ca_county#9, d_qoy#7, d_year#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(16) CometScan [native_comet] parquet spark_catalog.default.store_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -177,7 +177,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] Condition : isnotnull(ss_addr_sk#11) -(18) CometScan [native_comet] parquet spark_catalog.default.date_dim +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -236,7 +236,7 @@ Left output [3]: [ca_county#9, d_year#6, store_sales#22] Right output [2]: [ca_county#19, store_sales#21] Arguments: [ca_county#9], [ca_county#19], Inner, BuildRight -(31) CometScan [native_comet] parquet spark_catalog.default.store_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -248,7 +248,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_addr_sk#23) -(33) CometScan [native_comet] parquet spark_catalog.default.date_dim +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -311,7 +311,7 @@ Arguments: [ca_county#19], [ca_county#31], Inner, BuildRight Input [7]: [ca_county#9, d_year#6, store_sales#22, ca_county#19, store_sales#21, ca_county#31, store_sales#33] Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33] -(47) CometScan [native_comet] parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] Batched: true Location: InMemoryFileIndex [] @@ -370,7 +370,7 @@ Left output [5]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_s Right output [2]: [ca_county#42, web_sales#44] Arguments: [ca_county#9], [ca_county#42], Inner, BuildRight -(60) CometScan [native_comet] parquet spark_catalog.default.web_sales +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -433,7 +433,7 @@ Arguments: [ca_county#42], [ca_county#53], Inner, (CASE WHEN (web_sales#44 > 0.0 Input [9]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, ca_county#53, web_sales#55] Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55] -(74) CometScan [native_comet] parquet spark_catalog.default.web_sales +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -513,10 +513,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dyn BroadcastExchange (94) +- * CometColumnarToRow (93) +- CometFilter (92) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (91) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (91) -(91) CometScan [native_comet] parquet spark_catalog.default.date_dim +(91) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -538,10 +538,10 @@ Subquery:2 Hosting operator id = 16 Hosting Expression = ss_sold_date_sk#13 IN d BroadcastExchange (98) +- * CometColumnarToRow (97) +- CometFilter (96) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (95) -(95) CometScan [native_comet] parquet spark_catalog.default.date_dim +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -563,10 +563,10 @@ Subquery:3 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN d BroadcastExchange (102) +- * CometColumnarToRow (101) +- CometFilter (100) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (99) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (99) -(99) CometScan [native_comet] parquet spark_catalog.default.date_dim +(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt index 21535c42f5..435da991de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt @@ -19,20 +19,20 @@ WholeStageCodegen (1) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [ca_address_sk,ca_county] #5 CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #6 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #7 @@ -42,17 +42,17 @@ WholeStageCodegen (1) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,store_sales] #10 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] @@ -63,17 +63,17 @@ WholeStageCodegen (1) CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #12 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #14 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] @@ -84,7 +84,7 @@ WholeStageCodegen (1) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year,d_qoy] #4 ReusedExchange [ca_address_sk,ca_county] #5 @@ -97,7 +97,7 @@ WholeStageCodegen (1) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #9 ReusedExchange [ca_address_sk,ca_county] #5 @@ -110,7 +110,7 @@ WholeStageCodegen (1) CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk,d_year,d_qoy] #13 ReusedExchange [ca_address_sk,ca_county] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt index 08a421e82a..b27d40f91b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,15 +23,15 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -71,7 +71,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -(9) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] Condition : isnotnull(cs_item_sk#7) -(11) CometScan [native_comet] parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_comet] parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt index 6f783863c1..a89a381578 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt @@ -11,7 +11,7 @@ WholeStageCodegen (1) CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,sum,count,avg(UnscaledValue(cs_ext_discount_amt))] @@ -32,10 +32,10 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt index db7a5610b3..b78621c4bb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt @@ -15,23 +15,23 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- CometHashAggregate (48) : +- CometExchange (47) : +- CometHashAggregate (46) @@ -42,17 +42,17 @@ : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometFilter (29) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (28) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (28) : : : +- ReusedExchange (30) : : +- ReusedExchange (33) : +- CometBroadcastExchange (43) : +- CometBroadcastHashJoin (42) : :- CometFilter (37) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (36) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (36) : +- CometBroadcastExchange (41) : +- CometProject (40) : +- CometFilter (39) - : +- CometScan [native_comet] parquet spark_catalog.default.item (38) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (38) +- CometHashAggregate (62) +- CometExchange (61) +- CometHashAggregate (60) @@ -63,13 +63,13 @@ : :- CometProject (53) : : +- CometBroadcastHashJoin (52) : : :- CometFilter (50) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) : : +- ReusedExchange (51) : +- ReusedExchange (54) +- ReusedExchange (57) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -81,7 +81,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_manufact_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_category#13, i_manufact_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -198,7 +198,7 @@ Input [2]: [i_manufact_id#12, sum#15] Keys [1]: [i_manufact_id#12] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] @@ -234,7 +234,7 @@ Arguments: [cs_bill_addr_sk#16], [ca_address_sk#22], Inner, BuildRight Input [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, ca_address_sk#22] Arguments: [cs_item_sk#17, cs_ext_sales_price#18], [cs_item_sk#17, cs_ext_sales_price#18] -(36) CometScan [native_comet] parquet spark_catalog.default.item +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#23, i_manufact_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -245,7 +245,7 @@ ReadSchema: struct Input [2]: [i_item_sk#23, i_manufact_id#24] Condition : isnotnull(i_item_sk#23) -(38) CometScan [native_comet] parquet spark_catalog.default.item +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_category#13, i_manufact_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -295,7 +295,7 @@ Input [2]: [i_manufact_id#24, sum#25] Keys [1]: [i_manufact_id#24] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] -(49) CometScan [native_comet] parquet spark_catalog.default.web_sales +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] @@ -390,10 +390,10 @@ BroadcastExchange (73) +- * CometColumnarToRow (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (69) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_comet] parquet spark_catalog.default.date_dim +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt index 4b75691a13..cd8521b4eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,23 +24,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_manufact_id] #6 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #7 CometProject [i_manufact_id] [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_manufact_id] #8 CometHashAggregate [i_manufact_id,sum,cs_ext_sales_price] @@ -51,18 +51,18 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 CometBroadcastExchange [i_item_sk,i_manufact_id] #9 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #10 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_manufact_id] #11 CometHashAggregate [i_manufact_id,sum,ws_ext_sales_price] @@ -73,7 +73,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt index b4dd66bb54..ceab9d3f0a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt @@ -15,26 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt index 238061487f..a7c1215d1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt index e410139855..f01bb79a29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt @@ -15,40 +15,40 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (13) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) : : : +- ReusedExchange (14) : : +- BroadcastExchange (25) : : +- * CometColumnarToRow (24) : : +- CometProject (23) : : +- CometBroadcastHashJoin (22) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (20) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) : : +- ReusedExchange (21) : +- BroadcastExchange (33) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,14 +59,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#20] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18, ca_state#20] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt index 0d595eb78c..f1fe09fb46 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt index 075857a6b9..ae1d4c242b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt @@ -16,22 +16,22 @@ TakeOrderedAndProject (30) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) +- CometBroadcastExchange (18) +- CometProject (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct= 68.00)) A Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan [native_comet] parquet spark_catalog.default.inventory +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +71,7 @@ Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#13, cs_sold_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt index 660b5c5de8..7ab646a628 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt @@ -14,11 +14,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -26,11 +26,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [cs_item_sk] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt index bb790ceb71..591fa114fb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt @@ -14,15 +14,15 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : +- CometBroadcastExchange (29) : +- CometHashAggregate (28) : +- CometExchange (27) @@ -32,7 +32,7 @@ : :- CometProject (22) : : +- CometBroadcastHashJoin (21) : : :- CometFilter (19) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (18) : : +- ReusedExchange (20) : +- ReusedExchange (23) +- CometBroadcastExchange (42) @@ -44,12 +44,12 @@ :- CometProject (35) : +- CometBroadcastHashJoin (34) : :- CometFilter (32) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (31) : +- ReusedExchange (33) +- ReusedExchange (36) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +61,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -89,7 +89,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_comet] parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -131,7 +131,7 @@ Input [3]: [c_last_name#11, c_first_name#10, d_date#5] Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] Functions: [] -(18) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -190,7 +190,7 @@ Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] Right output [3]: [c_last_name#19, c_first_name#18, d_date#16] Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)], LeftSemi, BuildRight -(31) CometScan [native_comet] parquet spark_catalog.default.web_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -276,10 +276,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt index fb925a86b6..c9ecc235c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -23,15 +23,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometExchange [c_last_name,c_first_name,d_date] #7 @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 @@ -54,7 +54,7 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt index 6321c844f4..8e6d9265d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt @@ -15,17 +15,17 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (13) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -39,16 +39,16 @@ : :- CometProject (28) : : +- CometBroadcastHashJoin (27) : : :- CometFilter (25) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (24) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (26) : +- ReusedExchange (29) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ ReadSchema: struct 0.000000)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -205,7 +205,7 @@ Condition : (isnotnull(c_customer_sk#28) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] Arguments: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14], [c_customer_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#32, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#34, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#35, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -230,7 +230,7 @@ Arguments: [c_customer_sk#28], [ss_customer_sk#36], Inner, BuildRight Input [14]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -277,7 +277,7 @@ Left output [2]: [customer_id#26, year_total#27] Right output [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] Arguments: [customer_id#26], [customer_id#47], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -292,7 +292,7 @@ Condition : (isnotnull(c_customer_sk#55) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Arguments: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68], [c_customer_sk#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#56, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#57, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#58, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#59, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#61, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#62, 50, true, false, true) AS c_email_address#68] -(39) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -360,7 +360,7 @@ Arguments: [customer_id#26], [customer_id#80], Inner, BuildRight Input [12]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, customer_id#80, year_total#81] Arguments: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81], [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -375,7 +375,7 @@ Condition : (isnotnull(c_customer_sk#82) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] Arguments: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68], [c_customer_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#83, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#84, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#85, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#86, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#87, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#88, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#89, 50, true, false, true) AS c_email_address#68] -(57) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] Batched: true Location: InMemoryFileIndex [] @@ -439,7 +439,7 @@ Arguments: [customer_id#26], [customer_id#101], Inner, (CASE WHEN (year_total#81 Input [13]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81, customer_id#101, year_total#102] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] -(71) CometScan [native_comet] parquet spark_catalog.default.customer +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -454,7 +454,7 @@ Condition : (isnotnull(c_customer_sk#103) AND isnotnull(staticinvoke(class org.a Input [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] Arguments: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116], [c_customer_sk#103, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#104, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#105, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#106, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#107, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#109, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#110, 50, true, false, true) AS c_email_address#116] -(74) CometScan [native_comet] parquet spark_catalog.default.web_sales +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] Batched: true Location: InMemoryFileIndex [] @@ -522,7 +522,7 @@ Arguments: [customer_id#26], [customer_id#128], Inner, BuildRight Input [12]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, customer_id#128, year_total#129] Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] -(89) CometScan [native_comet] parquet spark_catalog.default.customer +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -537,7 +537,7 @@ Condition : (isnotnull(c_customer_sk#130) AND isnotnull(staticinvoke(class org.a Input [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] Arguments: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116], [c_customer_sk#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#131, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#132, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#133, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#134, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#135, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#136, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#137, 50, true, false, true) AS c_email_address#116] -(92) CometScan [native_comet] parquet spark_catalog.default.web_sales +(92) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] Batched: true Location: InMemoryFileIndex [] @@ -614,10 +614,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dy BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometFilter (109) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_comet] parquet spark_catalog.default.date_dim +(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -639,10 +639,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#41 IN d BroadcastExchange (115) +- * CometColumnarToRow (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt index 8e74ad0f85..47e4022f45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt @@ -21,20 +21,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 @@ -45,20 +45,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] @@ -70,10 +70,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -86,10 +86,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 CometBroadcastExchange [customer_id,year_total] #16 @@ -103,10 +103,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #19 @@ -119,9 +119,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt index c83736dfb2..00e2b8fbc1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt @@ -15,26 +15,26 @@ : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_comet] parquet spark_catalog.default.item (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) +- CometBroadcastExchange (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -54,7 +54,7 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -86,7 +86,7 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -(12) CometScan [native_comet] parquet spark_catalog.default.warehouse +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#11, w_state#12] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -114,7 +114,7 @@ Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -(18) CometScan [native_comet] parquet spark_catalog.default.item +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -142,7 +142,7 @@ Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -193,10 +193,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dyn BroadcastExchange (37) +- * CometColumnarToRow (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt index 56f45f3550..23945dce83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt @@ -16,27 +16,27 @@ WholeStageCodegen (1) CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] CometExchange [cr_order_number,cr_item_sk] #4 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometBroadcastExchange [w_warehouse_sk,w_state] #5 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #7 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt index d03780331b..3b1ad984a3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,10 +17,10 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometScan [native_comet] parquet spark_catalog.default.item (4) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -35,7 +35,7 @@ Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_ma Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometScan [native_comet] parquet spark_catalog.default.item +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt index b568d3ebc1..bff847188c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] @@ -18,4 +18,4 @@ WholeStageCodegen (1) CometHashAggregate [i_manufact,count] CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometScan [native_comet] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt index b709094800..183fffdf00 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt index d8fb182c5c..9de0f79dee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 CometProject [i_category] [i_item_sk,i_category_id,i_category] CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt index 0e66c78acb..62ddd93781 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.store (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometScan [native_comet] parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt index 738d423a98..c7445ea60c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [d_day_name] [d_date_sk,d_day_name] CometFilter [d_date_sk,d_year,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt index 79c146b0a2..134ab7939e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt @@ -19,7 +19,7 @@ TakeOrderedAndProject (34) : : : +- CometHashAggregate (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- * Sort (21) : : +- * Project (20) : : +- * Filter (19) @@ -31,11 +31,11 @@ TakeOrderedAndProject (34) : +- * CometColumnarToRow (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_comet] parquet spark_catalog.default.item (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) +- ReusedExchange (31) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -131,7 +131,7 @@ Join condition: None Output [3]: [item_sk#7, rnk#11, item_sk#12] Input [4]: [item_sk#7, rnk#11, item_sk#12, rnk#14] -(24) CometScan [native_comet] parquet spark_catalog.default.item +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_product_name#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -189,10 +189,10 @@ Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (38) +- CometProject (37) +- CometFilter (36) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (35) -(35) CometScan [native_comet] parquet spark_catalog.default.store_sales +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt index 82d9eb9726..70452e4c3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt @@ -28,13 +28,13 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] CometProject [ss_store_sk,ss_net_profit] CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] CometExchange [ss_item_sk] #2 CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] CometProject [ss_item_sk,ss_net_profit] CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (4) Sort [rnk] @@ -54,6 +54,6 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] InputAdapter CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt index 9f239bc5e4..9d3cf3d382 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt @@ -17,30 +17,30 @@ TakeOrderedAndProject (39) : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.item (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) +- BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.item (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -52,7 +52,7 @@ ReadSchema: struct= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometScan [native_comet] parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -66,7 +66,7 @@ Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -94,7 +94,7 @@ Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(14) CometScan [native_comet] parquet spark_catalog.default.customer_address +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -122,7 +122,7 @@ Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH, Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(20) CometScan [native_comet] parquet spark_catalog.default.date_dim +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -174,10 +174,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt index 408dad4982..b456a3d4a3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -21,19 +21,19 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk] #3 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt index 3f4034ad0f..338f5f4201 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt @@ -23,14 +23,14 @@ : : :- CometBroadcastExchange (4) : : : +- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometProject (7) : : +- CometFilter (6) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) :- * Project (49) : +- * Filter (48) : +- Window (47) @@ -49,10 +49,10 @@ : : :- CometBroadcastExchange (30) : : : +- CometProject (29) : : : +- CometFilter (28) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (27) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) : : +- CometProject (33) : : +- CometFilter (32) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) : +- ReusedExchange (36) +- * Project (72) +- * Filter (71) @@ -72,14 +72,14 @@ : :- CometBroadcastExchange (53) : : +- CometProject (52) : : +- CometFilter (51) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (50) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) : +- CometProject (56) : +- CometFilter (55) - : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (54) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) +- ReusedExchange (59) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -99,7 +99,7 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometScan [native_comet] parquet spark_catalog.default.web_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -123,7 +123,7 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -196,7 +196,7 @@ Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -(27) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] @@ -216,7 +216,7 @@ Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, c Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -(31) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -297,7 +297,7 @@ Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -(50) CometScan [native_comet] parquet spark_catalog.default.store_sales +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -317,7 +317,7 @@ Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -(54) CometScan [native_comet] parquet spark_catalog.default.store_returns +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -430,10 +430,10 @@ BroadcastExchange (83) +- * CometColumnarToRow (82) +- CometProject (81) +- CometFilter (80) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (79) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) -(79) CometScan [native_comet] parquet spark_catalog.default.date_dim +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt index 5e8cf98267..43ce5f13af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt @@ -32,7 +32,7 @@ WholeStageCodegen (11) CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) @@ -40,14 +40,14 @@ WholeStageCodegen (11) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] WholeStageCodegen (6) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -72,11 +72,11 @@ WholeStageCodegen (11) CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] @@ -102,9 +102,9 @@ WholeStageCodegen (11) CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt index 07fbe00126..f2dce3f729 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt @@ -17,18 +17,18 @@ TakeOrderedAndProject (74) : : :- CometUnion (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.store (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) :- * HashAggregate (43) : +- * CometColumnarToRow (42) : +- CometExchange (41) @@ -40,15 +40,15 @@ TakeOrderedAndProject (74) : : :- CometUnion (30) : : : :- CometProject (26) : : : : +- CometFilter (25) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (24) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) : : : +- CometProject (29) : : : +- CometFilter (28) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (27) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) : : +- ReusedExchange (31) : +- CometBroadcastExchange (37) : +- CometProject (36) : +- CometFilter (35) - : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (34) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) +- * HashAggregate (67) +- * CometColumnarToRow (66) +- CometExchange (65) @@ -60,22 +60,22 @@ TakeOrderedAndProject (74) : :- CometUnion (54) : : :- CometProject (46) : : : +- CometFilter (45) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (44) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) : : +- CometProject (53) : : +- CometBroadcastHashJoin (52) : : :- CometBroadcastExchange (48) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (47) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) : : +- CometProject (51) : : +- CometFilter (50) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (49) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) : +- ReusedExchange (55) +- CometBroadcastExchange (61) +- CometProject (60) +- CometFilter (59) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (58) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -91,7 +91,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -111,7 +111,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_comet] parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -139,7 +139,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_comet] parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -186,7 +186,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt# Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#37, store channel AS channel#38, concat(store, s_store_id#26) AS id#39] -(24) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -202,7 +202,7 @@ Condition : isnotnull(cs_catalog_page_sk#40) Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] -(27) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -234,7 +234,7 @@ Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -(34) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -281,7 +281,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#75, catalog channel AS channel#76, concat(catalog_page, cp_catalog_page_id#64) AS id#77] -(44) CometScan [native_comet] parquet spark_catalog.default.web_sales +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Batched: true Location: InMemoryFileIndex [] @@ -297,7 +297,7 @@ Condition : isnotnull(ws_web_site_sk#78) Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] -(47) CometScan [native_comet] parquet spark_catalog.default.web_returns +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] Batched: true Location: InMemoryFileIndex [] @@ -308,7 +308,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -123,7 +123,7 @@ Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometC Input [3]: [item_sk#9, d_date#6, cume_sales#11] Arguments: [item_sk#9, d_date#6, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] -(19) CometScan [native_comet] parquet spark_catalog.default.store_sales +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -227,10 +227,10 @@ BroadcastExchange (46) +- * CometColumnarToRow (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) -(42) CometScan [native_comet] parquet spark_catalog.default.date_dim +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt index bccebf349e..30c94463ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt @@ -27,7 +27,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) @@ -35,11 +35,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] CometColumnarExchange [item_sk,d_date] #7 WholeStageCodegen (4) @@ -57,6 +57,6 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt index c6e2fd4218..a05651a422 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt index 74381cf6d8..18bcdb314f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt index 8c17f9f8c6..2fc38bf838 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -45,7 +45,7 @@ Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcha Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt index 6b78c94e2f..b8716d034f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt @@ -20,10 +20,10 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manufact_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -31,11 +31,11 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [d_date_sk,d_qoy] #5 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [s_store_sk] #6 CometFilter [s_store_sk] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt index d2b2d2a277..247d1d2d5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt @@ -27,39 +27,39 @@ : : : : : : :- CometUnion (7) : : : : : : : :- CometProject (3) : : : : : : : : +- CometFilter (2) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : +- CometProject (6) : : : : : : : +- CometFilter (5) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (4) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (4) : : : : : : +- CometBroadcastExchange (11) : : : : : : +- CometProject (10) : : : : : : +- CometFilter (9) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) : : : : : +- CometBroadcastExchange (17) : : : : : +- CometProject (16) : : : : : +- CometFilter (15) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : : : : +- CometBroadcastExchange (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (30) : : : +- CometFilter (29) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) : : +- CometBroadcastExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) : +- CometBroadcastExchange (42) : +- CometProject (41) : +- CometFilter (40) - : +- CometScan [native_comet] parquet spark_catalog.default.store (39) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) +- CometBroadcastExchange (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (45) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -75,7 +75,7 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Arguments: [sold_date_sk#5, customer_sk#6, item_sk#7], [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] -(4) CometScan [native_comet] parquet spark_catalog.default.web_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -95,7 +95,7 @@ Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] -(8) CometScan [native_comet] parquet spark_catalog.default.item +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_class#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -123,7 +123,7 @@ Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -151,7 +151,7 @@ Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] Arguments: [customer_sk#6], [customer_sk#6] -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#20, c_current_addr_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -189,7 +189,7 @@ Input [2]: [c_customer_sk#20, c_current_addr_sk#21] Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] Functions: [] -(28) CometScan [native_comet] parquet spark_catalog.default.store_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] @@ -214,7 +214,7 @@ Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -(33) CometScan [native_comet] parquet spark_catalog.default.customer_address +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -242,7 +242,7 @@ Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -(39) CometScan [native_comet] parquet spark_catalog.default.store +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_county#30, s_state#31] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -270,7 +270,7 @@ Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildR Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -(45) CometScan [native_comet] parquet spark_catalog.default.date_dim +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -340,10 +340,10 @@ BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometProject (61) +- CometFilter (60) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (59) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) -(59) CometScan [native_comet] parquet spark_catalog.default.date_dim +(59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -372,10 +372,10 @@ BroadcastExchange (68) +- * CometColumnarToRow (67) +- CometProject (66) +- CometFilter (65) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (64) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) -(64) CometScan [native_comet] parquet spark_catalog.default.date_dim +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_month_seq#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -404,10 +404,10 @@ Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (72) +- CometProject (71) +- CometFilter (70) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (69) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) -(69) CometScan [native_comet] parquet spark_catalog.default.date_dim +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#44, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -446,10 +446,10 @@ Subquery:5 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (79) +- CometProject (78) +- CometFilter (77) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (76) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) -(76) CometScan [native_comet] parquet spark_catalog.default.date_dim +(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#44, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt index 29a7c6c022..ea21e4f051 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometUnion [sold_date_sk,customer_sk,item_sk] CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,25 +36,25 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [i_item_sk] #5 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #9 WholeStageCodegen (1) @@ -71,7 +71,7 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #4 WholeStageCodegen (1) CometColumnarToRow @@ -81,19 +81,19 @@ WholeStageCodegen (1) CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] CometBroadcastExchange [s_county,s_state] #13 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [(d_month_seq + 1)] #3 ReusedSubquery [(d_month_seq + 3)] #4 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt index 4a8c312ba7..47e0117481 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt index 5634c77a32..dde0a8ebc7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt @@ -11,11 +11,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt index ff1e432989..5562e86b91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt @@ -15,24 +15,24 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -43,7 +43,7 @@ : : :- CometProject (33) : : : +- CometBroadcastHashJoin (32) : : : :- CometFilter (30) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (29) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (31) : : +- ReusedExchange (34) : +- ReusedExchange (37) @@ -57,13 +57,13 @@ : :- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometFilter (44) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (43) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) : : +- ReusedExchange (45) : +- ReusedExchange (48) +- ReusedExchange (51) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_color#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -196,7 +196,7 @@ Input [2]: [i_item_id#16, sum#17] Keys [1]: [i_item_id#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(29) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -258,7 +258,7 @@ Input [2]: [i_item_id#26, sum#27] Keys [1]: [i_item_id#26] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -(43) CometScan [native_comet] parquet spark_catalog.default.web_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (63) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_comet] parquet spark_catalog.default.date_dim +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt index 682980d345..88b9477d1d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_id,i_color] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] @@ -52,7 +52,7 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt index bee4d6ae52..eb21a3abd7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache Input [3]: [i_item_sk#1, i_brand#2, i_category#3] Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_comet] parquet spark_catalog.default.call_center +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt index 027e80ca6b..b131b85ea9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt @@ -28,23 +28,23 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt index 88fe551ea0..85c8622ee4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt @@ -14,16 +14,16 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- CometBroadcastExchange (38) : +- CometFilter (37) @@ -35,13 +35,13 @@ : :- CometProject (25) : : +- CometBroadcastHashJoin (24) : : :- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (21) : : +- ReusedExchange (23) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometBroadcastHashJoin (29) : :- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- ReusedExchange (28) +- CometBroadcastExchange (53) +- CometFilter (52) @@ -53,12 +53,12 @@ :- CometProject (45) : +- CometBroadcastHashJoin (44) : :- CometFilter (42) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (41) : +- ReusedExchange (43) +- ReusedExchange (46) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -152,7 +152,7 @@ Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] Input [2]: [item_id#12, ss_item_rev#13] Condition : isnotnull(ss_item_rev#13) -(21) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -176,7 +176,7 @@ Arguments: [cs_item_sk#14], [i_item_sk#18], Inner, BuildRight Input [5]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_sk#18, i_item_id#19] Arguments: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19], [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_date#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -243,7 +243,7 @@ Arguments: [item_id#12], [item_id#23], Inner, ((((cast(ss_item_rev#13 as decimal Input [4]: [item_id#12, ss_item_rev#13, item_id#23, cs_item_rev#24] Arguments: [item_id#12, ss_item_rev#13, cs_item_rev#24], [item_id#12, ss_item_rev#13, cs_item_rev#24] -(41) CometScan [native_comet] parquet spark_catalog.default.web_sales +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] @@ -325,14 +325,14 @@ BroadcastExchange (67) +- CometProject (65) +- CometBroadcastHashJoin (64) :- CometFilter (59) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) +- CometBroadcastExchange (63) +- CometProject (62) +- CometFilter (61) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (60) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (60) -(58) CometScan [native_comet] parquet spark_catalog.default.date_dim +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -343,7 +343,7 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(60) CometScan [native_comet] parquet spark_catalog.default.date_dim +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -382,10 +382,10 @@ Subquery:2 Hosting operator id = 61 Hosting Expression = Subquery scalar-subquer * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_comet] parquet spark_catalog.default.date_dim +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -409,14 +409,14 @@ BroadcastExchange (81) +- CometProject (79) +- CometBroadcastHashJoin (78) :- CometFilter (73) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) +- CometBroadcastExchange (77) +- CometProject (76) +- CometFilter (75) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) -(72) CometScan [native_comet] parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_date#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -427,7 +427,7 @@ ReadSchema: struct Input [2]: [d_date_sk#20, d_date#21] Condition : isnotnull(d_date_sk#20) -(74) CometScan [native_comet] parquet spark_catalog.default.date_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt index f53195bf27..8524266dc5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt @@ -15,7 +15,7 @@ WholeStageCodegen (1) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -24,7 +24,7 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] [d_date] CometFilter [d_date,d_week_seq] @@ -34,17 +34,17 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cs_item_rev] #6 CometFilter [item_id,cs_item_rev] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #8 WholeStageCodegen (1) @@ -65,18 +65,18 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #9 CometProject [d_date] CometFilter [d_date,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] ReusedExchange [i_item_sk,i_item_id] #4 CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #9 CometBroadcastExchange [item_id,ws_item_rev] #11 CometFilter [item_id,ws_item_rev] @@ -88,7 +88,7 @@ WholeStageCodegen (1) CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [i_item_sk,i_item_id] #4 ReusedExchange [d_date_sk] #10 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt index 3660644e29..6d7e26d609 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt @@ -13,19 +13,19 @@ : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (15) : : +- CometProject (14) : : +- CometFilter (13) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (12) : +- CometBroadcastExchange (21) : +- CometProject (20) : +- CometFilter (19) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) +- CometBroadcastExchange (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -36,14 +36,14 @@ : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_comet] parquet spark_catalog.default.store (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -55,7 +55,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -97,7 +97,7 @@ Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, s Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(12) CometScan [native_comet] parquet spark_catalog.default.store +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -125,7 +125,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] -(18) CometScan [native_comet] parquet spark_catalog.default.date_dim +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -161,7 +161,7 @@ Input [9]: [d_week_seq#5, ss_store_sk#1, sum#38, sum#39, sum#40, sum#41, sum#42, Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(26) CometScan [native_comet] parquet spark_catalog.default.store +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#45, s_store_id#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -189,7 +189,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#45], Inner, BuildRight Input [11]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#45, s_store_id#47] Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47] -(32) CometScan [native_comet] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#48, d_week_seq#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt index 4f08c8c75d..d6ad4a9650 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] @@ -37,8 +37,8 @@ WholeStageCodegen (1) CometBroadcastExchange [s_store_sk,s_store_id] #6 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [d_week_seq] #7 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt index 4c79e9addf..b0905506ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt @@ -15,22 +15,22 @@ : : : +- CometBroadcastHashJoin (7) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) : : : +- CometBroadcastExchange (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) : : +- CometBroadcastExchange (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) : +- CometBroadcastExchange (17) : +- CometProject (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) +- CometBroadcastExchange (32) +- CometProject (31) +- CometBroadcastHashJoin (30) :- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.item (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) +- CometBroadcastExchange (29) +- CometFilter (28) +- CometHashAggregate (27) @@ -38,10 +38,10 @@ +- CometHashAggregate (25) +- CometProject (24) +- CometFilter (23) - +- CometScan [native_comet] parquet spark_catalog.default.item (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) -(1) CometScan [native_comet] parquet spark_catalog.default.customer_address +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -56,7 +56,7 @@ Condition : isnotnull(ca_address_sk#1) Input [2]: [ca_address_sk#1, ca_state#2] Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) CometScan [native_comet] parquet spark_catalog.default.customer +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -80,7 +80,7 @@ Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(9) CometScan [native_comet] parquet spark_catalog.default.store_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -105,7 +105,7 @@ Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(14) CometScan [native_comet] parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -133,7 +133,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(20) CometScan [native_comet] parquet spark_catalog.default.item +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -144,7 +144,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_comet] parquet spark_catalog.default.item +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_comet] parquet spark_catalog.default.date_dim +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#11, d_year#24, d_moy#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt index 135f7fd10f..35d3166cef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt @@ -16,13 +16,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_category,sum,count,i_current_price] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt index 6871295754..e5e165b5ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt @@ -15,24 +15,24 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan [native_comet] parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- CometHashAggregate (42) : +- CometExchange (41) : +- CometHashAggregate (40) @@ -43,7 +43,7 @@ : : :- CometProject (33) : : : +- CometBroadcastHashJoin (32) : : : :- CometFilter (30) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (29) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (31) : : +- ReusedExchange (34) : +- ReusedExchange (37) @@ -57,13 +57,13 @@ : :- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometFilter (44) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (43) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) : : +- ReusedExchange (45) : +- ReusedExchange (48) +- ReusedExchange (51) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -75,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#11, i_item_id#12] Condition : isnotnull(i_item_sk#11) -(17) CometScan [native_comet] parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#13, i_category#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -196,7 +196,7 @@ Input [2]: [i_item_id#16, sum#17] Keys [1]: [i_item_id#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(29) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -258,7 +258,7 @@ Input [2]: [i_item_id#26, sum#27] Keys [1]: [i_item_id#26] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -(43) CometScan [native_comet] parquet spark_catalog.default.web_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -353,10 +353,10 @@ BroadcastExchange (67) +- * CometColumnarToRow (66) +- CometProject (65) +- CometFilter (64) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (63) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) -(63) CometScan [native_comet] parquet spark_catalog.default.date_dim +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt index 8ff5e75747..914f53e34b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,24 +24,24 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #6 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [i_item_id] #7 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_id,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] @@ -52,7 +52,7 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 ReusedExchange [ca_address_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt index 16ecc9f3dd..4f090653c8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt @@ -18,30 +18,30 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.promotion (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.item (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) +- BroadcastExchange (63) +- * CometColumnarToRow (62) +- CometHashAggregate (61) @@ -58,7 +58,7 @@ : : : :- CometProject (46) : : : : +- CometBroadcastHashJoin (45) : : : : :- CometFilter (43) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (42) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (42) : : : : +- ReusedExchange (44) : : : +- ReusedExchange (47) : : +- ReusedExchange (50) @@ -66,7 +66,7 @@ +- ReusedExchange (56) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -78,7 +78,7 @@ ReadSchema: struct Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(5) CometScan [native_comet] parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -123,7 +123,7 @@ Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -(18) CometScan [native_comet] parquet spark_catalog.default.item +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -151,7 +151,7 @@ Arguments: [ss_item_sk#3], [i_item_sk#12], Inner, BuildRight Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11, i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] Arguments: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -(24) CometScan [native_comet] parquet spark_catalog.default.store_sales +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt index 747b2dbf52..2e8a46fe81 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] CometHashAggregate [ss_store_sk,ss_item_sk,revenue,sum,sum(UnscaledValue(ss_sales_price))] @@ -18,7 +18,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -26,15 +26,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #5 CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometBroadcastExchange [ss_store_sk,ave] #6 CometFilter [ss_store_sk,ave] CometHashAggregate [ss_store_sk,ave,sum,count,avg(revenue)] @@ -46,6 +46,6 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt index 645515cf6d..7599e9ecd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt @@ -17,22 +17,22 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.ship_mode (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) +- CometHashAggregate (45) +- CometExchange (44) +- CometHashAggregate (43) @@ -45,14 +45,14 @@ : : :- CometProject (33) : : : +- CometBroadcastHashJoin (32) : : : :- CometFilter (30) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (29) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) : : : +- ReusedExchange (31) : : +- ReusedExchange (34) : +- ReusedExchange (37) +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -105,7 +105,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ Right keys [1]: [ws_bill_customer_sk#10] Join type: LeftAnti Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -173,7 +173,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(28) CometScan [native_comet] parquet spark_catalog.default.customer_address +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -205,7 +205,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#2] Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16] -(35) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -269,10 +269,10 @@ BroadcastExchange (51) +- * CometColumnarToRow (50) +- CometProject (49) +- CometFilter (48) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (47) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) -(47) CometScan [native_comet] parquet spark_catalog.default.date_dim +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt index c68549d28f..68bf32c40b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt @@ -17,11 +17,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -29,11 +29,11 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -41,7 +41,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -51,7 +51,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -61,7 +61,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -69,4 +69,4 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha InputAdapter CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt index e8b9572c77..0e32283618 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt @@ -13,26 +13,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) +- CometBroadcastExchange (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_comet] parquet spark_catalog.default.promotion (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -44,7 +44,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -92,7 +92,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_comet] parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -106,7 +106,7 @@ Condition : isnotnull(s_store_sk#7) (12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#7, s_county#8, s_state#9] -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -118,7 +118,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#7, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -274,10 +274,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt index aeefdda422..ea88d5805a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt @@ -23,7 +23,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -31,11 +31,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -44,7 +44,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #6 WholeStageCodegen (2) @@ -64,10 +64,10 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_net_profit,ss_sold_date_sk,s_state] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt index 3e03a29a60..9ecf9be21f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt @@ -12,33 +12,33 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) : : :- CometFilter (6) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (5) : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (13) : : +- ReusedExchange (15) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (19) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (20) +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.time_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -57,7 +57,7 @@ Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, s Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(5) CometScan [native_comet] parquet spark_catalog.default.web_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -69,7 +69,7 @@ ReadSchema: struct date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_comet] parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_comet] parquet spark_catalog.default.date_dim +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt index a89be99fc8..c770348a8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt index 538f78b8b3..2de3317bc8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt @@ -15,26 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) +- CometBroadcastExchange (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.customer (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +46,7 @@ ReadSchema: struct= 1) AND (cnt#17 <= 5)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt index 3c2f1f34f6..5d798743dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt index a8677af312..649b5dc4d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,10 +51,10 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) : +- ReusedExchange (44) +- CometBroadcastExchange (68) +- CometHashAggregate (67) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -88,7 +88,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -170,7 +170,7 @@ Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -257,7 +257,7 @@ Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] -(39) CometScan [native_comet] parquet spark_catalog.default.web_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -340,7 +340,7 @@ Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] -(57) CometScan [native_comet] parquet spark_catalog.default.web_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt index d4c76e024d..400468d3e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt @@ -17,20 +17,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,20 +41,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] @@ -66,10 +66,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -82,9 +82,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index dce46d3120..7afb1e3be6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -22,19 +22,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -44,14 +44,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -61,14 +61,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -88,11 +88,11 @@ : : :- CometProject (71) : : : +- CometBroadcastHashJoin (70) : : : :- CometFilter (68) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -104,7 +104,7 @@ : : :- CometProject (87) : : : +- CometBroadcastHashJoin (86) : : : :- CometFilter (84) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (83) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) : : : +- ReusedExchange (85) : : +- ReusedExchange (88) : +- CometSort (94) @@ -118,14 +118,14 @@ : :- CometProject (101) : : +- CometBroadcastHashJoin (100) : : :- CometFilter (98) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) : : +- ReusedExchange (99) : +- ReusedExchange (102) +- CometSort (108) +- ReusedExchange (107) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -85,7 +85,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometScan [native_comet] parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -113,7 +113,7 @@ Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] -(15) CometScan [native_comet] parquet spark_catalog.default.customer_address +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [1]: [ca_zip#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -127,7 +127,7 @@ Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVa Input [1]: [ca_zip#12] Arguments: [ca_zip#13], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) AS ca_zip#13] -(18) CometScan [native_comet] parquet spark_catalog.default.customer_address +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#14, ca_zip#15] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -142,7 +142,7 @@ Condition : isnotnull(ca_address_sk#14) Input [2]: [ca_address_sk#14, ca_zip#15] Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#15, 10, true, false, true) AS ca_zip#16] -(21) CometScan [native_comet] parquet spark_catalog.default.customer +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,10 +256,10 @@ BroadcastExchange (49) +- * CometColumnarToRow (48) +- CometProject (47) +- CometFilter (46) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (45) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) -(45) CometScan [native_comet] parquet spark_catalog.default.date_dim +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt index 2e68b4656c..50673ff9c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -20,15 +20,15 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometExchange [ca_zip] #6 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] CometBroadcastExchange [ca_zip] #7 CometProject [ca_zip] CometFilter [ca_zip,cnt] @@ -47,8 +47,8 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [c_current_addr_sk] #9 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt index 8010bde9b0..bbc01dd8aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt @@ -23,28 +23,28 @@ TakeOrderedAndProject (106) : : : : : :- CometSort (4) : : : : : : +- CometExchange (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : +- CometSort (9) : : : : : +- CometExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) : : : : +- CometBroadcastExchange (15) : : : : +- CometProject (14) : : : : +- CometFilter (13) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (12) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) : : : +- CometBroadcastExchange (21) : : : +- CometProject (20) : : : +- CometFilter (19) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) : : +- CometBroadcastExchange (27) : : +- CometProject (26) : : +- CometFilter (25) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (24) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) : +- CometBroadcastExchange (33) : +- CometProject (32) : +- CometFilter (31) - : +- CometScan [native_comet] parquet spark_catalog.default.promotion (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) :- * HashAggregate (69) : +- * CometColumnarToRow (68) : +- CometExchange (67) @@ -62,17 +62,17 @@ TakeOrderedAndProject (106) : : : : : :- CometSort (43) : : : : : : +- CometExchange (42) : : : : : : +- CometFilter (41) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (40) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) : : : : : +- CometSort (48) : : : : : +- CometExchange (47) : : : : : +- CometProject (46) : : : : : +- CometFilter (45) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (44) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) : : : : +- ReusedExchange (51) : : : +- CometBroadcastExchange (57) : : : +- CometProject (56) : : : +- CometFilter (55) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (54) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) : : +- ReusedExchange (60) : +- ReusedExchange (63) +- * HashAggregate (99) @@ -92,22 +92,22 @@ TakeOrderedAndProject (106) : : : : :- CometSort (73) : : : : : +- CometExchange (72) : : : : : +- CometFilter (71) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (70) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) : : : : +- CometSort (78) : : : : +- CometExchange (77) : : : : +- CometProject (76) : : : : +- CometFilter (75) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (74) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) : : : +- ReusedExchange (81) : : +- CometBroadcastExchange (87) : : +- CometProject (86) : : +- CometFilter (85) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_site (84) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) : +- ReusedExchange (90) +- ReusedExchange (93) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -127,7 +127,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -159,7 +159,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#1 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -(12) CometScan [native_comet] parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -187,7 +187,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -(18) CometScan [native_comet] parquet spark_catalog.default.store +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#16, s_store_id#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -215,7 +215,7 @@ Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(24) CometScan [native_comet] parquet spark_catalog.default.item +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_current_price#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -243,7 +243,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -(30) CometScan [native_comet] parquet spark_catalog.default.promotion +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [2]: [p_promo_sk#21, p_channel_tv#22] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -290,7 +290,7 @@ Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_r Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#33, store channel AS channel#34, concat(store, s_store_id#18) AS id#35] -(40) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] @@ -310,7 +310,7 @@ Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIR Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] -(44) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -354,7 +354,7 @@ Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -(54) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -425,7 +425,7 @@ Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_ Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#64, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#65, catalog channel AS channel#66, concat(catalog_page, cp_catalog_page_id#52) AS id#67] -(70) CometScan [native_comet] parquet spark_catalog.default.web_sales +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -445,7 +445,7 @@ Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIR Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] -(74) CometScan [native_comet] parquet spark_catalog.default.web_returns +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -489,7 +489,7 @@ Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -(84) CometScan [native_comet] parquet spark_catalog.default.web_site +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#82, web_site_id#83] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] @@ -598,10 +598,10 @@ BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometProject (109) +- CometFilter (108) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (107) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(107) CometScan [native_comet] parquet spark_catalog.default.date_dim +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt index 9bed5e4a3c..d403cf479b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometExchange [ss_item_sk,ss_ticket_number] #3 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,28 +36,28 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] WholeStageCodegen (2) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow @@ -77,18 +77,18 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometExchange [cs_item_sk,cs_order_number] #11 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan [native_comet] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 WholeStageCodegen (3) @@ -110,17 +110,17 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometExchange [ws_item_sk,ws_order_number] #15 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedExchange [d_date_sk] #6 CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] ReusedExchange [i_item_sk] #8 ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt index 05a0cdd321..8b66e63cf3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt @@ -16,15 +16,15 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : : +- CometBroadcastExchange (34) : : +- CometFilter (33) : : +- CometHashAggregate (32) @@ -38,20 +38,20 @@ : : :- CometProject (23) : : : +- CometBroadcastHashJoin (22) : : : :- CometFilter (20) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (19) : : : +- ReusedExchange (21) : : +- ReusedExchange (24) : +- CometBroadcastExchange (40) : +- CometProject (39) : +- CometFilter (38) - : +- CometScan [native_comet] parquet spark_catalog.default.customer (37) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) +- CometBroadcastExchange (46) +- CometProject (45) +- CometFilter (44) - +- CometScan [native_comet] parquet spark_catalog.default.customer_address (43) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -63,7 +63,7 @@ ReadSchema: struct= 62.00)) A Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan [native_comet] parquet spark_catalog.default.inventory +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +71,7 @@ Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan [native_comet] parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +103,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan [native_comet] parquet spark_catalog.default.store_sales +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#13, ss_sold_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt index 40fc82d84d..28d160a4a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt @@ -14,11 +14,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -26,11 +26,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [ss_item_sk] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt index 705db63a99..a700acceb7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt @@ -13,16 +13,16 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) : : :- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- ReusedExchange (11) : +- CometBroadcastExchange (36) : +- CometHashAggregate (35) @@ -33,13 +33,13 @@ : :- CometProject (24) : : +- CometBroadcastHashJoin (23) : : :- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (20) : : +- ReusedExchange (22) : +- CometBroadcastExchange (30) : +- CometProject (29) : +- CometBroadcastHashJoin (28) : :- CometFilter (26) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) : +- ReusedExchange (27) +- CometBroadcastExchange (50) +- CometHashAggregate (49) @@ -50,12 +50,12 @@ :- CometProject (43) : +- CometBroadcastHashJoin (42) : :- CometFilter (40) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (39) : +- ReusedExchange (41) +- ReusedExchange (44) -(1) CometScan [native_comet] parquet spark_catalog.default.store_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -67,7 +67,7 @@ ReadSchema: struct Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_item_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -95,7 +95,7 @@ Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,7 +145,7 @@ Input [2]: [i_item_id#7, sum#11] Keys [1]: [i_item_id#7] Functions [1]: [sum(sr_return_quantity#2)] -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -169,7 +169,7 @@ Arguments: [cr_item_sk#12], [i_item_sk#16], Inner, BuildRight Input [5]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14, i_item_sk#16, i_item_id#17] Arguments: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17], [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] -(25) CometScan [native_comet] parquet spark_catalog.default.date_dim +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -232,7 +232,7 @@ Arguments: [item_id#23], [item_id#21], Inner, BuildRight Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] -(39) CometScan [native_comet] parquet spark_catalog.default.web_returns +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] Batched: true Location: InMemoryFileIndex [] @@ -310,18 +310,18 @@ BroadcastExchange (68) +- CometProject (66) +- CometBroadcastHashJoin (65) :- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (64) +- CometProject (63) +- CometBroadcastHashJoin (62) - :- CometScan [native_comet] parquet spark_catalog.default.date_dim (57) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (57) +- CometBroadcastExchange (61) +- CometProject (60) +- CometFilter (59) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (58) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_date#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -332,13 +332,13 @@ ReadSchema: struct Input [2]: [d_date_sk#8, d_date#9] Condition : isnotnull(d_date_sk#8) -(57) CometScan [native_comet] parquet spark_catalog.default.date_dim +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(58) CometScan [native_comet] parquet spark_catalog.default.date_dim +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -391,15 +391,15 @@ BroadcastExchange (79) +- CometProject (77) +- CometBroadcastHashJoin (76) :- CometFilter (70) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (69) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometBroadcastHashJoin (73) - :- CometScan [native_comet] parquet spark_catalog.default.date_dim (71) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (71) +- ReusedExchange (72) -(69) CometScan [native_comet] parquet spark_catalog.default.date_dim +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,7 +410,7 @@ ReadSchema: struct Input [2]: [d_date_sk#18, d_date#19] Condition : isnotnull(d_date_sk#18) -(71) CometScan [native_comet] parquet spark_catalog.default.date_dim +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt index c986b86f49..825d31a9d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -23,24 +23,24 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #3 CometProject [d_date] [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cr_item_qty] #7 CometHashAggregate [item_id,cr_item_qty,i_item_id,sum,sum(cr_return_quantity)] @@ -51,7 +51,7 @@ WholeStageCodegen (1) CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #9 WholeStageCodegen (1) @@ -60,18 +60,18 @@ WholeStageCodegen (1) CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #10 CometProject [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] ReusedExchange [d_week_seq] #4 ReusedExchange [i_item_sk,i_item_id] #5 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date] #10 CometBroadcastExchange [item_id,wr_item_qty] #12 CometHashAggregate [item_id,wr_item_qty,i_item_id,sum,sum(wr_return_quantity)] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [i_item_sk,i_item_id] #5 ReusedExchange [d_date_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt index c907048b21..45c375a787 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt @@ -14,27 +14,27 @@ TakeOrderedAndProject (33) : : : : +- CometBroadcastHashJoin (8) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) : : : +- CometBroadcastExchange (12) : : : +- CometFilter (11) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) : +- CometBroadcastExchange (23) : +- CometProject (22) : +- CometFilter (21) - : +- CometScan [native_comet] parquet spark_catalog.default.income_band (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.store_returns (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -49,7 +49,7 @@ Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2) Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#1, 16, true, false, true) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#5, 20, true, false, true) AS c_first_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#6, 30, true, false, true) AS c_last_name#9] -(4) CometScan [native_comet] parquet spark_catalog.default.customer_address +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#10, ca_city#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -77,7 +77,7 @@ Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(10) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -101,7 +101,7 @@ Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(15) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -125,7 +125,7 @@ Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(20) CometScan [native_comet] parquet spark_catalog.default.income_band +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -157,7 +157,7 @@ Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_c Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(27) CometScan [native_comet] parquet spark_catalog.default.store_returns +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt index 6936907ec8..cc7cf901e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt @@ -15,21 +15,21 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometScan [native_comet] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt index 4bab741e14..c263f03d76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt @@ -20,36 +20,36 @@ : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometBroadcastExchange (3) : : : : : : : +- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_page (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometProject (16) : : : : +- CometFilter (15) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) +- CometBroadcastExchange (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_comet] parquet spark_catalog.default.reason (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -65,7 +65,7 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometScan [native_comet] parquet spark_catalog.default.web_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -89,7 +89,7 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(9) CometScan [native_comet] parquet spark_catalog.default.web_page +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -113,7 +113,7 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(14) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -141,7 +141,7 @@ Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_s Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -169,7 +169,7 @@ Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status# Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(26) CometScan [native_comet] parquet spark_catalog.default.customer_address +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -197,7 +197,7 @@ Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(32) CometScan [native_comet] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -225,7 +225,7 @@ Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -(38) CometScan [native_comet] parquet spark_catalog.default.reason +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#35, r_reason_desc#36] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -281,10 +281,10 @@ BroadcastExchange (53) +- * CometColumnarToRow (52) +- CometProject (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#33, d_year#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt index df11eb5798..52cf7b57a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -29,30 +29,30 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometBroadcastExchange [wp_web_page_sk] #4 CometFilter [wp_web_page_sk] - CometScan [native_comet] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #7 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt index c0054aaa80..1af5e2612c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt @@ -14,18 +14,18 @@ TakeOrderedAndProject (24) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +37,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -65,7 +65,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -141,10 +141,10 @@ BroadcastExchange (29) +- * CometColumnarToRow (28) +- CometProject (27) +- CometFilter (26) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) -(25) CometScan [native_comet] parquet spark_catalog.default.date_dim +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt index f595259476..92a7eb7e8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt @@ -17,7 +17,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -25,12 +25,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt index f2454e937d..fc6b12b834 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt @@ -15,15 +15,15 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : +- BroadcastExchange (31) : +- * CometColumnarToRow (30) : +- CometHashAggregate (29) @@ -34,7 +34,7 @@ : :- CometProject (23) : : +- CometBroadcastHashJoin (22) : : :- CometFilter (20) - : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (19) : : +- ReusedExchange (21) : +- ReusedExchange (24) +- BroadcastExchange (45) @@ -47,12 +47,12 @@ :- CometProject (37) : +- CometBroadcastHashJoin (36) : :- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : +- ReusedExchange (35) +- ReusedExchange (38) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -64,7 +64,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -92,7 +92,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] -(9) CometScan [native_comet] parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -137,7 +137,7 @@ Functions: [] (18) CometColumnarToRow [codegen id : 3] Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -(19) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -200,7 +200,7 @@ Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_ Join type: LeftAnti Join condition: None -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] @@ -295,10 +295,10 @@ BroadcastExchange (56) +- * CometColumnarToRow (55) +- CometProject (54) +- CometFilter (53) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (52) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) -(52) CometScan [native_comet] parquet spark_catalog.default.date_dim +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt index 6b09d49e07..f687139735 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt @@ -18,7 +18,7 @@ WholeStageCodegen (4) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -26,15 +26,15 @@ WholeStageCodegen (4) InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #4 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (1) @@ -48,7 +48,7 @@ WholeStageCodegen (4) CometProject [cs_bill_customer_sk,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 @@ -65,7 +65,7 @@ WholeStageCodegen (4) CometProject [ws_bill_customer_sk,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #4 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt index 6ff7198461..2e49d20af4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt @@ -18,19 +18,19 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) +: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (16) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) : : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) @@ -44,12 +44,12 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (26) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (32) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) : : : : : +- * CometColumnarToRow (65) @@ -64,12 +64,12 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (47) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (53) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) : : : : +- * CometColumnarToRow (86) @@ -84,12 +84,12 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (68) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (74) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) : : : +- * CometColumnarToRow (107) @@ -104,12 +104,12 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (89) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (95) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) : : +- * CometColumnarToRow (128) @@ -124,12 +124,12 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (110) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (116) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) : +- * CometColumnarToRow (149) @@ -144,12 +144,12 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (131) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (137) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) +- * CometColumnarToRow (170) @@ -164,16 +164,16 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (152) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (158) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -188,7 +188,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -216,7 +216,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -244,7 +244,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -288,7 +288,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometScan [native_comet] parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -315,7 +315,7 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometScan [native_comet] parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -379,7 +379,7 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometScan [native_comet] parquet spark_catalog.default.store_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -406,7 +406,7 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometScan [native_comet] parquet spark_catalog.default.time_dim +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -470,7 +470,7 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometScan [native_comet] parquet spark_catalog.default.store_sales +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -497,7 +497,7 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometScan [native_comet] parquet spark_catalog.default.time_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -561,7 +561,7 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometScan [native_comet] parquet spark_catalog.default.store_sales +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -588,7 +588,7 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometScan [native_comet] parquet spark_catalog.default.time_dim +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -652,7 +652,7 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometScan [native_comet] parquet spark_catalog.default.store_sales +(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -679,7 +679,7 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometScan [native_comet] parquet spark_catalog.default.time_dim +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -743,7 +743,7 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometScan [native_comet] parquet spark_catalog.default.store_sales +(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -770,7 +770,7 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometScan [native_comet] parquet spark_catalog.default.time_dim +(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -834,7 +834,7 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometScan [native_comet] parquet spark_catalog.default.store_sales +(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -861,7 +861,7 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometScan [native_comet] parquet spark_catalog.default.time_dim +(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt index c47266d366..3490714189 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt @@ -19,19 +19,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -48,12 +48,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 @@ -71,12 +71,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 @@ -94,12 +94,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 @@ -117,12 +117,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 @@ -140,12 +140,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 @@ -163,12 +163,12 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt index 15f4eace86..47ce0d4a91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan [native_comet] parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -45,7 +45,7 @@ Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarchar Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) AS i_class#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) AS i_category#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -170,10 +170,10 @@ BroadcastExchange (34) +- * CometColumnarToRow (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt index f7f05c899a..140fe297a1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt @@ -20,10 +20,10 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] CometFilter [i_item_sk,i_brand,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -31,11 +31,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt index 2f50da0bab..b3f32555de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt @@ -2,10 +2,10 @@ * Project (4) +- * CometColumnarToRow (3) +- CometFilter (2) - +- CometScan [native_comet] parquet spark_catalog.default.reason (1) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) -(1) CometScan [native_comet] parquet spark_catalog.default.reason +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -33,10 +33,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) -(5) CometScan [native_comet] parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -84,10 +84,10 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (13) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -135,10 +135,10 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -186,10 +186,10 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (29) -(29) CometScan [native_comet] parquet spark_catalog.default.store_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -237,10 +237,10 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) - +- CometScan [native_comet] parquet spark_catalog.default.store_sales (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (37) -(37) CometScan [native_comet] parquet spark_catalog.default.store_sales +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt index 8644223a0b..510cf60bd8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 @@ -23,7 +23,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 @@ -49,7 +49,7 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 @@ -62,10 +62,10 @@ WholeStageCodegen (1) CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 CometColumnarToRow InputAdapter CometFilter [r_reason_sk] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt index b3916111e7..08ae744b5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt @@ -13,19 +13,19 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometScan [native_comet] parquet spark_catalog.default.web_page (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometHashAggregate (43) @@ -39,16 +39,16 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -63,7 +63,7 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -91,7 +91,7 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -119,7 +119,7 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.web_page +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -163,7 +163,7 @@ Functions [1]: [count(1)] (25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometScan [native_comet] parquet spark_catalog.default.web_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -190,7 +190,7 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometScan [native_comet] parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt index 1a5ae5b626..2db3b6c957 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt @@ -14,19 +14,19 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometScan [native_comet] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt index ef109a81aa..0ba113dd4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt @@ -19,32 +19,32 @@ : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometProject (3) : : : : : : +- CometFilter (2) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometFilter (5) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) : : +- CometBroadcastExchange (23) : : +- CometProject (22) : : +- CometFilter (21) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) : +- CometBroadcastExchange (29) : +- CometProject (28) : +- CometFilter (27) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) -(1) CometScan [native_comet] parquet spark_catalog.default.call_center +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -59,7 +59,7 @@ Condition : isnotnull(cc_call_center_sk#1) Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cc_call_center_id#2, 16, true, false, true) AS cc_call_center_id#5, cc_name#3, cc_manager#4] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -84,7 +84,7 @@ Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -112,7 +112,7 @@ Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(15) CometScan [native_comet] parquet spark_catalog.default.customer +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -136,7 +136,7 @@ Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -(20) CometScan [native_comet] parquet spark_catalog.default.customer_address +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -164,7 +164,7 @@ Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -(26) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -192,7 +192,7 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -(32) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#25, hd_buy_potential#26] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -252,10 +252,10 @@ BroadcastExchange (48) +- * CometColumnarToRow (47) +- CometProject (46) +- CometFilter (45) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (44) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) -(44) CometScan [native_comet] parquet spark_catalog.default.date_dim +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt index 994ca92319..5be6489e2d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt @@ -20,10 +20,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -31,23 +31,23 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #7 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk] #9 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt index 5decd5c0d1..c317a12697 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,15 +23,15 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_manufact_id#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -71,7 +71,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -(9) CometScan [native_comet] parquet spark_catalog.default.web_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] Condition : isnotnull(ws_item_sk#7) -(11) CometScan [native_comet] parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -178,10 +178,10 @@ BroadcastExchange (35) +- * CometColumnarToRow (34) +- CometProject (33) +- CometFilter (32) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) -(31) CometScan [native_comet] parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt index 6be4fb9e4f..7a1825cc9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt @@ -11,7 +11,7 @@ WholeStageCodegen (1) CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -19,11 +19,11 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk] #3 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,sum,count,avg(UnscaledValue(ws_ext_discount_amt))] @@ -32,10 +32,10 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt index 498abd2426..799d860c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt @@ -11,19 +11,19 @@ : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometScan [native_comet] parquet spark_catalog.default.reason (12) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometScan [native_comet] parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -73,7 +73,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometScan [native_comet] parquet spark_catalog.default.reason +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt index 9b21bd273a..d761953c97 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt @@ -12,13 +12,13 @@ WholeStageCodegen (1) CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometExchange [ss_item_sk,ss_ticket_number] #2 CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometExchange [sr_item_sk,sr_ticket_number] #3 CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [r_reason_sk] #4 CometProject [r_reason_sk] CometFilter [r_reason_sk,r_reason_desc] - CometScan [native_comet] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt index 002c60f908..3caf1593da 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt @@ -19,30 +19,30 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometSort (9) : : : : +- CometExchange (8) : : : : +- CometProject (7) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (6) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) : : : +- CometSort (15) : : : +- CometExchange (14) : : : +- CometProject (13) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (12) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) : : +- CometBroadcastExchange (20) : : +- CometProject (19) : : +- CometFilter (18) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) : +- CometBroadcastExchange (26) : +- CometProject (25) : +- CometFilter (24) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (23) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) +- CometBroadcastExchange (32) +- CometProject (31) +- CometFilter (30) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -65,7 +65,7 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -92,7 +92,7 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometScan [native_comet] parquet spark_catalog.default.web_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -115,7 +115,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometScan [native_comet] parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -143,7 +143,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometScan [native_comet] parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -171,7 +171,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometScan [native_comet] parquet spark_catalog.default.web_site +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt index 956fe14b3e..068a535b6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt @@ -22,24 +22,24 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [wr_order_number] CometExchange [wr_order_number] #4 CometProject [wr_order_number] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt index 2926fcc7e3..b149119f5d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt @@ -18,14 +18,14 @@ : : : : : +- CometExchange (4) : : : : : +- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : +- CometProject (14) : : : : +- CometSortMergeJoin (13) : : : : :- CometSort (10) : : : : : +- CometExchange (9) : : : : : +- CometProject (8) : : : : : +- CometFilter (7) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) : : : : +- CometSort (12) : : : : +- ReusedExchange (11) : : : +- CometProject (28) @@ -34,7 +34,7 @@ : : : : +- CometExchange (19) : : : : +- CometProject (18) : : : : +- CometFilter (17) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (16) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) : : : +- CometProject (26) : : : +- CometSortMergeJoin (25) : : : :- CometSort (22) @@ -44,18 +44,18 @@ : : +- CometBroadcastExchange (33) : : +- CometProject (32) : : +- CometFilter (31) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : +- CometBroadcastExchange (39) : +- CometProject (38) : +- CometFilter (37) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (36) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) +- CometBroadcastExchange (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_comet] parquet spark_catalog.default.web_site (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -78,7 +78,7 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometScan [native_comet] parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -122,7 +122,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometScan [native_comet] parquet spark_catalog.default.web_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -182,7 +182,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometScan [native_comet] parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#15, d_date#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -210,7 +210,7 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#15], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#15] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_address +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -238,7 +238,7 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#17], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#17] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometScan [native_comet] parquet spark_catalog.default.web_site +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#19, web_company_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt index be2453ac89..6f07040b87 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt @@ -21,14 +21,14 @@ WholeStageCodegen (2) CometExchange [ws_order_number] #2 CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] CometExchange [ws_order_number] #3 CometProject [ws_warehouse_sk,ws_order_number] CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 CometProject [wr_order_number] @@ -37,7 +37,7 @@ WholeStageCodegen (2) CometExchange [wr_order_number] #4 CometProject [wr_order_number] CometFilter [wr_order_number,wr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] @@ -47,12 +47,12 @@ WholeStageCodegen (2) CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [web_site_sk] #7 CometProject [web_site_sk] CometFilter [web_site_sk,web_company_name] - CometScan [native_comet] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt index 86b62b1982..f90d8b12b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt @@ -11,22 +11,22 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan [native_comet] parquet spark_catalog.default.time_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometScan [native_comet] parquet spark_catalog.default.store (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -69,7 +69,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan [native_comet] parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -97,7 +97,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan [native_comet] parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt index 34e10d3a24..36f4009885 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan [native_comet] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt index 527d11b75e..420f37fccc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt @@ -11,29 +11,29 @@ : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (2) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) - :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (12) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) +- ReusedExchange (13) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] ReadSchema: struct -(2) CometScan [native_comet] parquet spark_catalog.default.date_dim +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -79,7 +79,7 @@ Functions: [] Input [2]: [customer_sk#7, item_sk#8] Arguments: [customer_sk#7, item_sk#8], [customer_sk#7 ASC NULLS FIRST, item_sk#8 ASC NULLS FIRST] -(12) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -149,10 +149,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt index f2cb24041e..49b28b94be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometHashAggregate [ss_customer_sk,ss_item_sk] CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -20,17 +20,17 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] CometExchange [cs_bill_customer_sk,cs_item_sk] #5 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt index 68a1ee0c6d..4cc725ef2b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt @@ -16,18 +16,18 @@ :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -67,7 +67,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -150,10 +150,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt index cb1b3359a9..f563efa55d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt @@ -21,7 +21,7 @@ WholeStageCodegen (3) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -29,12 +29,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt index cbaeb3a9ae..7e57135c10 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt @@ -13,24 +13,24 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : : +- CometBroadcastExchange (11) : : +- CometProject (10) : : +- CometFilter (9) - : : +- CometScan [native_comet] parquet spark_catalog.default.ship_mode (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) : +- CometBroadcastExchange (16) : +- CometFilter (15) - : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) +- CometBroadcastExchange (22) +- CometProject (21) +- CometFilter (20) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (19) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -41,7 +41,7 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -96,7 +96,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan [native_comet] parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] Arguments: [customer_sk#14], [ws_bill_customer_sk#10 AS customer_sk#14] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customer_sk#14], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_comet] parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_county#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(30) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt index e6c4ab5862..08c0ad906c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt @@ -13,11 +13,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -25,28 +25,28 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [customer_sk] #5 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometProject [cs_ship_customer_sk] [customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt index 1d9bfdb401..49c1c8e0d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,10 +51,10 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) : +- ReusedExchange (44) +- CometBroadcastExchange (68) +- CometHashAggregate (67) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -88,7 +88,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -170,7 +170,7 @@ Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Batched: true Location: InMemoryFileIndex [] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#23, year_total#24] Right output [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -257,7 +257,7 @@ Condition : (isnotnull(c_customer_sk#46) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] Arguments: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59], [c_customer_sk#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#47, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#48, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#49, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#50, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#52, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#53, 50, true, false, true) AS c_email_address#59] -(39) CometScan [native_comet] parquet spark_catalog.default.web_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] @@ -325,7 +325,7 @@ Arguments: [customer_id#23], [customer_id#68], Inner, BuildRight Input [9]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, customer_id#68, year_total#69] Arguments: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69], [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -340,7 +340,7 @@ Condition : (isnotnull(c_customer_sk#70) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Arguments: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59], [c_customer_sk#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#71, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#72, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#73, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#74, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#75, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#76, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#77, 50, true, false, true) AS c_email_address#59] -(57) CometScan [native_comet] parquet spark_catalog.default.web_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] Batched: true Location: InMemoryFileIndex [] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt index 1b7dc9294a..9d22d7d913 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt @@ -17,20 +17,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -41,20 +41,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -66,10 +66,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -82,9 +82,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt index 77d08a829f..410aecff1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt @@ -13,18 +13,18 @@ TakeOrderedAndProject (23) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt index 2d88b65482..ad8631ea1d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt index e5195b3ed7..ea550c6c74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt @@ -12,12 +12,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -28,25 +28,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -54,18 +54,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -77,16 +77,16 @@ : +- CometBroadcastHashJoin (70) : :- CometBroadcastHashJoin (68) : : :- CometFilter (66) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) : : +- ReusedExchange (67) : +- ReusedExchange (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -109,7 +109,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -132,7 +132,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -244,7 +244,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -389,7 +389,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) -(65) CometScan [native_comet] parquet spark_catalog.default.store_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -(72) CometScan [native_comet] parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -493,19 +493,19 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (98) :- CometProject (89) : +- CometBroadcastHashJoin (88) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (86) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) : +- ReusedExchange (87) :- CometProject (93) : +- CometBroadcastHashJoin (92) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (90) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) : +- ReusedExchange (91) +- CometProject (97) +- CometBroadcastHashJoin (96) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (94) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) +- ReusedExchange (95) -(86) CometScan [native_comet] parquet spark_catalog.default.store_sales +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] @@ -524,7 +524,7 @@ Arguments: [ss_sold_date_sk#77], [d_date_sk#79], Inner, BuildRight Input [4]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77, d_date_sk#79] Arguments: [quantity#80, list_price#81], [ss_quantity#75 AS quantity#80, ss_list_price#76 AS list_price#81] -(90) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -543,7 +543,7 @@ Arguments: [cs_sold_date_sk#84], [d_date_sk#86], Inner, BuildRight Input [4]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84, d_date_sk#86] Arguments: [quantity#87, list_price#88], [cs_quantity#82 AS quantity#87, cs_list_price#83 AS list_price#88] -(94) CometScan [native_comet] parquet spark_catalog.default.web_sales +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] Batched: true Location: InMemoryFileIndex [] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (103) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_comet] parquet spark_catalog.default.date_dim +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_comet] parquet spark_catalog.default.date_dim +(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#103] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (117) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_comet] parquet spark_catalog.default.date_dim +(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#65, d_week_seq#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (122) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_comet] parquet spark_catalog.default.date_dim +(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt index 0e0b2ea4b8..5d5557bc59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) @@ -75,26 +75,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -102,20 +102,20 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -128,7 +128,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #17 WholeStageCodegen (1) @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt index 9984111e07..a727c5e743 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt @@ -19,12 +19,12 @@ : : : +- CometBroadcastHashJoin (53) : : : :- CometBroadcastHashJoin (47) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (46) : : : : +- CometProject (45) : : : : +- CometBroadcastHashJoin (44) : : : : :- CometFilter (4) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) @@ -35,25 +35,25 @@ : : : : : :- CometProject (26) : : : : : : +- CometBroadcastHashJoin (25) : : : : : : :- CometFilter (6) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : : +- CometBroadcastExchange (24) : : : : : : +- CometBroadcastHashJoin (23) : : : : : : :- CometFilter (8) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : : +- CometBroadcastExchange (22) : : : : : : +- CometProject (21) : : : : : : +- CometBroadcastHashJoin (20) : : : : : : :- CometProject (15) : : : : : : : +- CometBroadcastHashJoin (14) : : : : : : : :- CometFilter (10) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : : +- CometBroadcastExchange (13) : : : : : : : +- CometFilter (12) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : : +- CometBroadcastExchange (19) : : : : : : +- CometProject (18) : : : : : : +- CometFilter (17) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : : +- ReusedExchange (27) : : : : +- CometBroadcastExchange (41) : : : : +- CometProject (40) @@ -61,18 +61,18 @@ : : : : :- CometProject (37) : : : : : +- CometBroadcastHashJoin (36) : : : : : :- CometFilter (34) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : : +- ReusedExchange (35) : : : : +- ReusedExchange (38) : : : +- CometBroadcastExchange (52) : : : +- CometBroadcastHashJoin (51) : : : :- CometFilter (49) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : : +- ReusedExchange (50) : : +- CometBroadcastExchange (58) : : +- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) : :- CometFilter (78) : : +- CometHashAggregate (77) : : +- CometExchange (76) @@ -83,7 +83,7 @@ : : : +- CometBroadcastHashJoin (70) : : : :- CometBroadcastHashJoin (68) : : : : :- CometFilter (66) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (65) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (65) : : : : +- ReusedExchange (67) : : : +- ReusedExchange (69) : : +- ReusedExchange (72) @@ -97,7 +97,7 @@ : : +- CometBroadcastHashJoin (84) : : :- CometBroadcastHashJoin (82) : : : :- CometFilter (80) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (79) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (79) : : : +- ReusedExchange (81) : : +- ReusedExchange (83) : +- ReusedExchange (86) @@ -123,7 +123,7 @@ +- ReusedExchange (112) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -135,7 +135,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -146,7 +146,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -169,7 +169,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -205,7 +205,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -281,7 +281,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -348,7 +348,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -380,7 +380,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -426,7 +426,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) -(65) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] Batched: true Location: InMemoryFileIndex [] @@ -488,7 +488,7 @@ Functions [2]: [sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), Input [6]: [channel#68, i_brand_id#61, i_class_id#62, i_category_id#63, sales#69, number_sales#70] Condition : (isnotnull(sales#69) AND (cast(sales#69 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) -(79) CometScan [native_comet] parquet spark_catalog.default.web_sales +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -695,22 +695,22 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (138) :- CometProject (126) : +- CometBroadcastHashJoin (125) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (123) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (123) : +- ReusedExchange (124) :- CometProject (133) : +- CometBroadcastHashJoin (132) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (127) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (127) : +- CometBroadcastExchange (131) : +- CometProject (130) : +- CometFilter (129) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (128) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (128) +- CometProject (137) +- CometBroadcastHashJoin (136) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (134) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (134) +- ReusedExchange (135) -(123) CometScan [native_comet] parquet spark_catalog.default.store_sales +(123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#143, ss_list_price#144, ss_sold_date_sk#145] Batched: true Location: InMemoryFileIndex [] @@ -729,14 +729,14 @@ Arguments: [ss_sold_date_sk#145], [d_date_sk#147], Inner, BuildRight Input [4]: [ss_quantity#143, ss_list_price#144, ss_sold_date_sk#145, d_date_sk#147] Arguments: [quantity#148, list_price#149], [ss_quantity#143 AS quantity#148, ss_list_price#144 AS list_price#149] -(127) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#150, cs_list_price#151, cs_sold_date_sk#152] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#152), dynamicpruningexpression(cs_sold_date_sk#152 IN dynamicpruning#153)] ReadSchema: struct -(128) CometScan [native_comet] parquet spark_catalog.default.date_dim +(128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#154, d_year#155] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -764,7 +764,7 @@ Arguments: [cs_sold_date_sk#152], [d_date_sk#154], Inner, BuildRight Input [4]: [cs_quantity#150, cs_list_price#151, cs_sold_date_sk#152, d_date_sk#154] Arguments: [quantity#156, list_price#157], [cs_quantity#150 AS quantity#156, cs_list_price#151 AS list_price#157] -(134) CometScan [native_comet] parquet spark_catalog.default.web_sales +(134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#158, ws_list_price#159, ws_sold_date_sk#160] Batched: true Location: InMemoryFileIndex [] @@ -812,10 +812,10 @@ BroadcastExchange (147) +- * CometColumnarToRow (146) +- CometProject (145) +- CometFilter (144) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (143) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) -(143) CometScan [native_comet] parquet spark_catalog.default.date_dim +(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#154, d_year#155] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -844,10 +844,10 @@ BroadcastExchange (152) +- * CometColumnarToRow (151) +- CometProject (150) +- CometFilter (149) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (148) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (148) -(148) CometScan [native_comet] parquet spark_catalog.default.date_dim +(148) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -874,10 +874,10 @@ BroadcastExchange (157) +- * CometColumnarToRow (156) +- CometProject (155) +- CometFilter (154) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (153) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (153) -(153) CometScan [native_comet] parquet spark_catalog.default.date_dim +(153) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#168] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt index 7f21febd85..047cf5ecfe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt @@ -21,12 +21,12 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #12 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #17 WholeStageCodegen (1) @@ -34,14 +34,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #4 ReusedExchange [d_date_sk] #18 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -53,7 +53,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -61,12 +61,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #5 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #6 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -77,7 +77,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) @@ -85,26 +85,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 CometProject [i_brand_id,i_class_id,i_category_id] @@ -112,19 +112,19 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #5 CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] @@ -136,7 +136,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #5 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 @@ -152,7 +152,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #5 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt index 6d8a27b1e5..72e9bd1b4a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt @@ -18,30 +18,30 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.item (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) :- CometHashAggregate (66) : +- CometExchange (65) : +- CometHashAggregate (64) @@ -58,14 +58,14 @@ : : : : : :- CometProject (45) : : : : : : +- CometBroadcastHashJoin (44) : : : : : : :- CometFilter (42) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (41) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (41) : : : : : : +- ReusedExchange (43) : : : : : +- ReusedExchange (46) : : : : +- ReusedExchange (49) : : : +- CometBroadcastExchange (55) : : : +- CometProject (54) : : : +- CometFilter (53) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (52) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (52) : : +- ReusedExchange (58) : +- ReusedExchange (61) :- CometHashAggregate (92) @@ -84,14 +84,14 @@ : : : : : :- CometProject (71) : : : : : : +- CometBroadcastHashJoin (70) : : : : : : :- CometFilter (68) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (67) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : : : : +- ReusedExchange (69) : : : : : +- ReusedExchange (72) : : : : +- ReusedExchange (75) : : : +- CometBroadcastExchange (81) : : : +- CometProject (80) : : : +- CometFilter (79) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (78) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (78) : : +- ReusedExchange (84) : +- ReusedExchange (87) :- CometHashAggregate (118) @@ -110,14 +110,14 @@ : : : : : :- CometProject (97) : : : : : : +- CometBroadcastHashJoin (96) : : : : : : :- CometFilter (94) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (93) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (93) : : : : : : +- ReusedExchange (95) : : : : : +- ReusedExchange (98) : : : : +- ReusedExchange (101) : : : +- CometBroadcastExchange (107) : : : +- CometProject (106) : : : +- CometFilter (105) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (104) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (104) : : +- ReusedExchange (110) : +- ReusedExchange (113) +- CometHashAggregate (143) @@ -136,7 +136,7 @@ : : : : :- CometProject (123) : : : : : +- CometBroadcastHashJoin (122) : : : : : :- CometFilter (120) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (119) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (119) : : : : : +- ReusedExchange (121) : : : : +- ReusedExchange (124) : : : +- ReusedExchange (127) @@ -144,10 +144,10 @@ : +- ReusedExchange (133) +- CometBroadcastExchange (138) +- CometFilter (137) - +- CometScan [native_comet] parquet spark_catalog.default.item (136) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (136) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -159,7 +159,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt index 75319785fa..51d992e9cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt index a480e94ee0..f85c898208 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt @@ -13,21 +13,21 @@ TakeOrderedAndProject (26) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) +- BroadcastExchange (18) +- * CometColumnarToRow (17) - +- CometScan [native_comet] parquet spark_catalog.default.warehouse (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -67,7 +67,7 @@ Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_pro (15) CometColumnarToRow [codegen id : 2] Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(16) CometScan [native_comet] parquet spark_catalog.default.warehouse +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output: [] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt index 08de1f27fb..221c6063ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] CometProject [inv_item_sk,inv_quantity_on_hand] CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -24,18 +24,18 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter - CometScan [native_comet] parquet spark_catalog.default.warehouse + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt index c808c0d20f..bf16cc4d21 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt @@ -14,18 +14,18 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) :- CometHashAggregate (29) : +- CometExchange (28) : +- CometHashAggregate (27) @@ -48,7 +48,7 @@ +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt index 420385984c..9ffdb57abc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt index 3861a964c3..b43ed9b192 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt @@ -15,40 +15,40 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (13) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) : : : +- ReusedExchange (14) : : +- BroadcastExchange (25) : : +- * CometColumnarToRow (24) : : +- CometProject (23) : : +- CometBroadcastHashJoin (22) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (20) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) : : +- ReusedExchange (21) : +- BroadcastExchange (33) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,14 +59,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#22] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt index 646bb805ec..fea6a32741 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt index d8984e74f9..e22a5c492c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt @@ -12,36 +12,36 @@ : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (12) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) +- CometBroadcastExchange (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -52,14 +52,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -96,7 +96,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan [native_comet] parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] Arguments: [customsk#14], [ws_bill_customer_sk#10 AS customsk#14] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customsk#14], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_comet] parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20, ca_state#22] Arguments: [c_current_cdemo_sk#2, ca_state#22], [c_current_cdemo_sk#2, ca_state#22] -(30) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt index 6a55cdf307..0aa670f8e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt @@ -13,11 +13,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -25,28 +25,28 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [customsk] #5 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt index fab15150f0..2562693a01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt @@ -19,19 +19,19 @@ TakeOrderedAndProject (43) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.store (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -44,7 +44,7 @@ TakeOrderedAndProject (43) +- ReusedExchange (29) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -56,7 +56,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -211,7 +211,7 @@ Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometC Input [3]: [item_sk#9, d_date#6, cume_sales#22] Arguments: [item_sk#9, d_date#6, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] -(31) CometScan [native_comet] parquet spark_catalog.default.store_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -418,10 +418,10 @@ BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometProject (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt index 1714124cc9..172baee830 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt @@ -40,7 +40,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #6 WholeStageCodegen (1) @@ -48,11 +48,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter BroadcastExchange #8 WholeStageCodegen (3) @@ -89,7 +89,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #7 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt index 02e411b04f..d1daee98c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache Input [3]: [i_item_sk#1, i_brand#2, i_category#3] Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_comet] parquet spark_catalog.default.call_center +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt index 9ac1e20eec..7df61c1995 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt @@ -28,23 +28,23 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt index 606941840f..162121609a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt @@ -21,18 +21,18 @@ : : : :- CometUnion (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) : :- * HashAggregate (43) : : +- * CometColumnarToRow (42) : : +- CometExchange (41) @@ -44,15 +44,15 @@ : : : :- CometUnion (30) : : : : :- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) : : : : +- CometProject (29) : : : : +- CometFilter (28) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (27) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) : : : +- ReusedExchange (31) : : +- CometBroadcastExchange (37) : : +- CometProject (36) : : +- CometFilter (35) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) : +- * HashAggregate (67) : +- * CometColumnarToRow (66) : +- CometExchange (65) @@ -64,19 +64,19 @@ : : :- CometUnion (54) : : : :- CometProject (46) : : : : +- CometFilter (45) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (44) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) : : : +- CometProject (53) : : : +- CometBroadcastHashJoin (52) : : : :- CometBroadcastExchange (48) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (47) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) : : : +- CometProject (51) : : : +- CometFilter (50) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) : : +- ReusedExchange (55) : +- CometBroadcastExchange (61) : +- CometProject (60) : +- CometFilter (59) - : +- CometScan [native_comet] parquet spark_catalog.default.web_site (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) :- * HashAggregate (79) : +- * CometColumnarToRow (78) : +- CometColumnarExchange (77) @@ -93,7 +93,7 @@ +- ReusedExchange (80) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -109,7 +109,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -129,7 +129,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_comet] parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -157,7 +157,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_comet] parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -204,7 +204,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt# Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] Results [5]: [store channel AS channel#35, concat(store, s_store_id#26) AS id#36, MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#39] -(24) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -220,7 +220,7 @@ Condition : isnotnull(cs_catalog_page_sk#40) Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] -(27) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -252,7 +252,7 @@ Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -(34) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -299,7 +299,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] Results [5]: [catalog channel AS channel#73, concat(catalog_page, cp_catalog_page_id#64) AS id#74, MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#75, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#76, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#77] -(44) CometScan [native_comet] parquet spark_catalog.default.web_sales +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Batched: true Location: InMemoryFileIndex [] @@ -315,7 +315,7 @@ Condition : isnotnull(ws_web_site_sk#78) Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] -(47) CometScan [native_comet] parquet spark_catalog.default.web_returns +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] Batched: true Location: InMemoryFileIndex [] @@ -326,7 +326,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_comet] parquet spark_catalog.default.item +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_current_price#17, i_category#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_comet] parquet spark_catalog.default.date_dim +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt index 5c979b2233..9e3765b84c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt @@ -16,13 +16,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_category,sum,count,i_current_price] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt index 27e71a746b..1990c685ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt @@ -46,10 +46,10 @@ : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,51 +62,51 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) : : : : : : : : : : : : : : +- CometBroadcastExchange (39) : : : : : : : : : : : : : : +- CometProject (38) : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) : : : : : : : : : : : : : +- CometBroadcastExchange (44) : : : : : : : : : : : : : +- CometFilter (43) - : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) : : : : : : : : : : : : +- CometBroadcastExchange (49) : : : : : : : : : : : : +- CometFilter (48) - : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) : : : : : : : : : : : +- ReusedExchange (52) : : : : : : : : : : +- CometBroadcastExchange (58) : : : : : : : : : : +- CometProject (57) : : : : : : : : : : +- CometFilter (56) - : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) : : : : : : : : : +- ReusedExchange (61) : : : : : : : : +- CometBroadcastExchange (66) : : : : : : : : +- CometFilter (65) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.promotion (64) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) : : : : : : : +- CometBroadcastExchange (71) : : : : : : : +- CometFilter (70) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (69) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) : : : : : : +- ReusedExchange (74) : : : : : +- CometBroadcastExchange (80) : : : : : +- CometProject (79) : : : : : +- CometFilter (78) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (77) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) : : : : +- ReusedExchange (83) : : : +- CometBroadcastExchange (88) : : : +- CometFilter (87) - : : : +- CometScan [native_comet] parquet spark_catalog.default.income_band (86) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) : : +- ReusedExchange (91) : +- CometBroadcastExchange (97) : +- CometProject (96) : +- CometFilter (95) - : +- CometScan [native_comet] parquet spark_catalog.default.item (94) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) +- CometSort (171) +- CometExchange (170) +- CometHashAggregate (169) @@ -149,10 +149,10 @@ : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (110) : : : : : : : : : : : : : : : : :- CometBroadcastExchange (106) : : : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (104) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (107) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) : : : : : : : : : : : : : : : +- CometSort (118) : : : : : : : : : : : : : : : +- CometProject (117) : : : : : : : : : : : : : : : +- CometFilter (116) @@ -160,7 +160,7 @@ : : : : : : : : : : : : : : : +- ReusedExchange (114) : : : : : : : : : : : : : : +- CometBroadcastExchange (123) : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (121) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) : : : : : : : : : : : : : +- ReusedExchange (126) : : : : : : : : : : : : +- ReusedExchange (129) : : : : : : : : : : : +- ReusedExchange (132) @@ -177,7 +177,7 @@ +- ReusedExchange (165) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -193,7 +193,7 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -225,7 +225,7 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -248,7 +248,7 @@ Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -315,7 +315,7 @@ Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometScan [native_comet] parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -339,7 +339,7 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -(36) CometScan [native_comet] parquet spark_catalog.default.store +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -367,7 +367,7 @@ Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -(42) CometScan [native_comet] parquet spark_catalog.default.customer +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -391,7 +391,7 @@ Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -(47) CometScan [native_comet] parquet spark_catalog.default.date_dim +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -427,7 +427,7 @@ Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(55) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#48, cd_marital_status#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -467,7 +467,7 @@ Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_stat Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(64) CometScan [native_comet] parquet spark_catalog.default.promotion +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#53] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -491,7 +491,7 @@ Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(69) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -527,7 +527,7 @@ Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -(77) CometScan [native_comet] parquet spark_catalog.default.customer_address +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -567,7 +567,7 @@ Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(86) CometScan [native_comet] parquet spark_catalog.default.income_band +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#70] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -603,7 +603,7 @@ Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(94) CometScan [native_comet] parquet spark_catalog.default.item +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -649,7 +649,7 @@ Arguments: hashpartitioning(item_sk#82, store_name#83, store_zip#84, 5), ENSURE_ Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] Arguments: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97], [item_sk#82 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, store_zip#84 ASC NULLS FIRST] -(104) CometScan [native_comet] parquet spark_catalog.default.store_sales +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] Batched: true Location: InMemoryFileIndex [] @@ -665,7 +665,7 @@ Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(107) CometScan [native_comet] parquet spark_catalog.default.store_returns +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -726,7 +726,7 @@ Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(121) CometScan [native_comet] parquet spark_catalog.default.date_dim +(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -962,10 +962,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dy BroadcastExchange (180) +- * CometColumnarToRow (179) +- CometFilter (178) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (177) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) -(177) CometScan [native_comet] parquet spark_catalog.default.date_dim +(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -987,10 +987,10 @@ Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN BroadcastExchange (184) +- * CometColumnarToRow (183) +- CometFilter (182) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (181) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) -(181) CometScan [native_comet] parquet spark_catalog.default.date_dim +(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt index 0c9153cd68..1dc9f0e3a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt @@ -47,17 +47,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -70,51 +70,51 @@ WholeStageCodegen (1) CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #12 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #12 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #13 CometBroadcastExchange [p_promo_sk] #14 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometBroadcastExchange [ib_income_band_sk] #17 CometFilter [ib_income_band_sk] - CometScan [native_comet] parquet spark_catalog.default.income_band [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] ReusedExchange [ib_income_band_sk] #17 CometBroadcastExchange [i_item_sk,i_product_name] #18 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] @@ -157,17 +157,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #22 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -175,7 +175,7 @@ WholeStageCodegen (1) ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 CometBroadcastExchange [d_date_sk,d_year] #23 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #10 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 ReusedExchange [d_date_sk,d_year] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt index 6d954b0a2d..46f670a0bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt @@ -20,19 +20,19 @@ TakeOrderedAndProject (74) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -75,7 +75,7 @@ TakeOrderedAndProject (74) +- ReusedExchange (59) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -87,7 +87,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -110,7 +110,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_comet] parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -124,7 +124,7 @@ Condition : isnotnull(s_store_sk#7) (12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#7, s_county#8, s_state#9] -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -136,7 +136,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -378,10 +378,10 @@ BroadcastExchange (71) +- * CometColumnarToRow (70) +- CometProject (69) +- CometFilter (68) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (67) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (67) -(67) CometScan [native_comet] parquet spark_catalog.default.date_dim +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt index bfa92f4dae..614e2c3014 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,11 +36,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) @@ -49,7 +49,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (2) @@ -70,12 +70,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_net_profit,ss_sold_date_sk,s_state] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [s_store_sk,s_state] #9 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #5 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt index 16ce320133..ee3242f2b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt @@ -27,45 +27,45 @@ : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (3) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (8) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : : : : : : : +- CometBroadcastExchange (15) : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (13) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) : : : : : : +- CometBroadcastExchange (21) : : : : : : +- CometProject (20) : : : : : : +- CometFilter (19) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (18) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) : : : : : +- CometBroadcastExchange (27) : : : : : +- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) : : : : +- CometBroadcastExchange (33) : : : : +- CometProject (32) : : : : +- CometFilter (31) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : : : +- CometBroadcastExchange (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) : : +- CometBroadcastExchange (43) : : +- CometFilter (42) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) : +- CometBroadcastExchange (48) : +- CometFilter (47) - : +- CometScan [native_comet] parquet spark_catalog.default.promotion (46) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) +- CometSort (57) +- CometExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (53) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -77,7 +77,7 @@ ReadSchema: struct date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_comet] parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_comet] parquet spark_catalog.default.date_dim +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt index a89be99fc8..c770348a8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt index 000db1f74c..b0ae8bb695 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,10 +51,10 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) : +- ReusedExchange (44) +- CometBroadcastExchange (68) +- CometHashAggregate (67) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -88,7 +88,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -170,7 +170,7 @@ Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -257,7 +257,7 @@ Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] -(39) CometScan [native_comet] parquet spark_catalog.default.web_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -340,7 +340,7 @@ Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] -(57) CometScan [native_comet] parquet spark_catalog.default.web_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt index d4c76e024d..400468d3e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt @@ -17,20 +17,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,20 +41,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] @@ -66,10 +66,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -82,9 +82,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt index 1547bac90e..6925eb301e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt @@ -22,19 +22,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -44,14 +44,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -61,14 +61,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -88,11 +88,11 @@ : : :- CometProject (71) : : : +- CometBroadcastHashJoin (70) : : : :- CometFilter (68) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -104,7 +104,7 @@ : : :- CometProject (87) : : : +- CometBroadcastHashJoin (86) : : : :- CometFilter (84) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (83) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) : : : +- ReusedExchange (85) : : +- ReusedExchange (88) : +- CometSort (94) @@ -118,14 +118,14 @@ : :- CometProject (101) : : +- CometBroadcastHashJoin (100) : : :- CometFilter (98) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) : : +- ReusedExchange (99) : +- ReusedExchange (102) +- CometSort (108) +- ReusedExchange (107) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -78,7 +78,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -213,10 +213,10 @@ BroadcastExchange (42) +- * CometColumnarToRow (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) -(38) CometScan [native_comet] parquet spark_catalog.default.date_dim +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt index d976ac8da8..9c46912086 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -28,15 +28,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #6 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] CometExchange [i_category] #7 CometHashAggregate [i_category,sum,isEmpty,total_sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt index 91c8e15e33..96a0ec392e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt @@ -15,18 +15,18 @@ :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt index 25073f98c7..196e3744a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt @@ -20,7 +20,7 @@ WholeStageCodegen (3) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -28,12 +28,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt index ad676c2153..7e11c91157 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt @@ -12,36 +12,36 @@ : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (12) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) +- CometBroadcastExchange (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -52,14 +52,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -96,7 +96,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan [native_comet] parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] Arguments: [customer_sk#14], [ws_bill_customer_sk#10 AS customer_sk#14] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customer_sk#14], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_comet] parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_county#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(30) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt index e6c4ab5862..08c0ad906c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt @@ -13,11 +13,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -25,28 +25,28 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [customer_sk] #5 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometProject [cs_ship_customer_sk] [customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt index 3523612197..444da6c14b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt @@ -17,13 +17,13 @@ TakeOrderedAndProject (79) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- BroadcastExchange (38) : : +- * HashAggregate (37) : : +- * CometColumnarToRow (36) @@ -36,13 +36,13 @@ TakeOrderedAndProject (79) : : : +- CometBroadcastHashJoin (26) : : : :- CometProject (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (25) : : : +- CometFilter (24) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : +- CometBroadcastExchange (30) : : +- CometFilter (29) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (28) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (28) : +- BroadcastExchange (57) : +- * Filter (56) : +- * HashAggregate (55) @@ -56,10 +56,10 @@ TakeOrderedAndProject (79) : : +- CometBroadcastHashJoin (46) : : :- CometProject (42) : : : +- CometFilter (41) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (40) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (40) : : +- CometBroadcastExchange (45) : : +- CometFilter (44) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (43) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) : +- ReusedExchange (48) +- BroadcastExchange (76) +- * HashAggregate (75) @@ -73,14 +73,14 @@ TakeOrderedAndProject (79) : +- CometBroadcastHashJoin (66) : :- CometProject (62) : : +- CometFilter (61) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (60) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (60) : +- CometBroadcastExchange (65) : +- CometFilter (64) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (63) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (63) +- ReusedExchange (68) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -95,7 +95,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharC Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -120,7 +120,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -172,7 +172,7 @@ Results [2]: [c_customer_id#9 AS customer_id#25, MakeDecimal(sum(UnscaledValue(( Input [2]: [customer_id#25, year_total#26] Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.00)) -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -187,7 +187,7 @@ Condition : (isnotnull(c_customer_sk#27) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Arguments: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)) AS c_customer_id#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#29, 20)) AS c_first_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#30, 30)) AS c_last_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#31, 1)) AS c_preferred_cust_flag#38, c_birth_country#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#33, 13)) AS c_login#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#34, 50)) AS c_email_address#40] -(23) CometScan [native_comet] parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] @@ -212,7 +212,7 @@ Arguments: [c_customer_sk#27], [ss_customer_sk#41], Inner, BuildRight Input [12]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44] Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_list_price#43, ss_sold_date_sk#44] -(28) CometScan [native_comet] parquet spark_catalog.default.date_dim +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#46, d_year#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -270,7 +270,7 @@ Right keys [1]: [customer_id#50] Join type: Inner Join condition: None -(40) CometScan [native_comet] parquet spark_catalog.default.customer +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -285,7 +285,7 @@ Condition : (isnotnull(c_customer_sk#55) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] Arguments: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68], [c_customer_sk#55, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#56, 16)) AS c_customer_id#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#57, 20)) AS c_first_name#64, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#58, 30)) AS c_last_name#65, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#59, 1)) AS c_preferred_cust_flag#66, c_birth_country#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#61, 13)) AS c_login#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#62, 50)) AS c_email_address#68] -(43) CometScan [native_comet] parquet spark_catalog.default.web_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] @@ -364,7 +364,7 @@ Join condition: None Output [8]: [customer_id#25, year_total#26, customer_id#50, customer_first_name#51, customer_last_name#52, customer_email_address#53, year_total#54, year_total#80] Input [9]: [customer_id#25, year_total#26, customer_id#50, customer_first_name#51, customer_last_name#52, customer_email_address#53, year_total#54, customer_id#79, year_total#80] -(60) CometScan [native_comet] parquet spark_catalog.default.customer +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#81, c_customer_id#82, c_first_name#83, c_last_name#84, c_preferred_cust_flag#85, c_birth_country#86, c_login#87, c_email_address#88] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -379,7 +379,7 @@ Condition : (isnotnull(c_customer_sk#81) AND isnotnull(static_invoke(CharVarchar Input [8]: [c_customer_sk#81, c_customer_id#82, c_first_name#83, c_last_name#84, c_preferred_cust_flag#85, c_birth_country#86, c_login#87, c_email_address#88] Arguments: [c_customer_sk#81, c_customer_id#89, c_first_name#90, c_last_name#91, c_preferred_cust_flag#92, c_birth_country#86, c_login#93, c_email_address#94], [c_customer_sk#81, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#82, 16)) AS c_customer_id#89, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#83, 20)) AS c_first_name#90, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#84, 30)) AS c_last_name#91, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#85, 1)) AS c_preferred_cust_flag#92, c_birth_country#86, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#87, 13)) AS c_login#93, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#88, 50)) AS c_email_address#94] -(63) CometScan [native_comet] parquet spark_catalog.default.web_sales +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#95, ws_ext_discount_amt#96, ws_ext_list_price#97, ws_sold_date_sk#98] Batched: true Location: InMemoryFileIndex [] @@ -464,10 +464,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (83) +- * CometColumnarToRow (82) +- CometFilter (81) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (80) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (80) -(80) CometScan [native_comet] parquet spark_catalog.default.date_dim +(80) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -489,10 +489,10 @@ Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#44 IN d BroadcastExchange (87) +- * CometColumnarToRow (86) +- CometFilter (85) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (84) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (84) -(84) CometScan [native_comet] parquet spark_catalog.default.date_dim +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#46, d_year#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt index ab06e2b485..d1bfad7c74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt @@ -20,20 +20,20 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -51,20 +51,20 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) @@ -83,10 +83,10 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 InputAdapter @@ -106,9 +106,9 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt index 6516d9f2e9..119f33af19 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt @@ -15,18 +15,18 @@ TakeOrderedAndProject (25) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -148,10 +148,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt index 22a37e1494..f288464255 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,12 +30,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt index 70928a5cfd..c4d1ce45d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt @@ -13,12 +13,12 @@ TakeOrderedAndProject (88) : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -29,25 +29,25 @@ TakeOrderedAndProject (88) : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -55,18 +55,18 @@ TakeOrderedAndProject (88) : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- BroadcastExchange (86) +- * Filter (85) +- * HashAggregate (84) @@ -80,16 +80,16 @@ TakeOrderedAndProject (88) : +- CometBroadcastHashJoin (72) : :- CometBroadcastHashJoin (70) : : :- CometFilter (68) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (67) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (67) : : +- ReusedExchange (69) : +- ReusedExchange (71) +- CometBroadcastExchange (77) +- CometProject (76) +- CometFilter (75) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -101,7 +101,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -112,7 +112,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -135,7 +135,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -171,7 +171,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -247,7 +247,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -314,7 +314,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -346,7 +346,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -402,7 +402,7 @@ Results [6]: [store AS channel#55, i_brand_id#40, i_class_id#41, i_category_id#4 Input [6]: [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57] Condition : (isnotnull(sales#56) AND (cast(sales#56 as decimal(32,6)) > cast(Subquery scalar-subquery#58, [id=#59] as decimal(32,6)))) -(67) CometScan [native_comet] parquet spark_catalog.default.store_sales +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] @@ -434,7 +434,7 @@ Arguments: [ss_item_sk#60], [i_item_sk#66], Inner, BuildRight Input [8]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] Arguments: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69], [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69] -(74) CometScan [native_comet] parquet spark_catalog.default.date_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -515,19 +515,19 @@ Subquery:1 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquer +- CometUnion (101) :- CometProject (92) : +- CometBroadcastHashJoin (91) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (89) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) : +- ReusedExchange (90) :- CometProject (96) : +- CometBroadcastHashJoin (95) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (93) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (93) : +- ReusedExchange (94) +- CometProject (100) +- CometBroadcastHashJoin (99) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (97) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) +- ReusedExchange (98) -(89) CometScan [native_comet] parquet spark_catalog.default.store_sales +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] @@ -546,7 +546,7 @@ Arguments: [ss_sold_date_sk#87], [d_date_sk#89], Inner, BuildRight Input [4]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87, d_date_sk#89] Arguments: [quantity#90, list_price#91], [ss_quantity#85 AS quantity#90, ss_list_price#86 AS list_price#91] -(93) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94] Batched: true Location: InMemoryFileIndex [] @@ -565,7 +565,7 @@ Arguments: [cs_sold_date_sk#94], [d_date_sk#96], Inner, BuildRight Input [4]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94, d_date_sk#96] Arguments: [quantity#97, list_price#98], [cs_quantity#92 AS quantity#97, cs_list_price#93 AS list_price#98] -(97) CometScan [native_comet] parquet spark_catalog.default.web_sales +(97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] @@ -624,10 +624,10 @@ BroadcastExchange (111) +- * CometColumnarToRow (110) +- CometProject (109) +- CometFilter (108) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (107) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(107) CometScan [native_comet] parquet spark_catalog.default.date_dim +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -655,10 +655,10 @@ Subquery:7 Hosting operator id = 107 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -681,10 +681,10 @@ BroadcastExchange (120) +- * CometColumnarToRow (119) +- CometProject (118) +- CometFilter (117) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (116) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (116) -(116) CometScan [native_comet] parquet spark_catalog.default.date_dim +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#116] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -721,10 +721,10 @@ BroadcastExchange (125) +- * CometColumnarToRow (124) +- CometProject (123) +- CometFilter (122) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (121) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) -(121) CometScan [native_comet] parquet spark_catalog.default.date_dim +(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -752,10 +752,10 @@ Subquery:16 Hosting operator id = 121 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (129) +- CometProject (128) +- CometFilter (127) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (126) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (126) -(126) CometScan [native_comet] parquet spark_catalog.default.date_dim +(126) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt index 8d14d0d6cf..812bf0d7dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt @@ -15,17 +15,17 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] @@ -42,7 +42,7 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -51,19 +51,19 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -74,7 +74,7 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) @@ -82,26 +82,26 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -109,20 +109,20 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 InputAdapter BroadcastExchange #15 @@ -143,7 +143,7 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #17 WholeStageCodegen (1) @@ -152,19 +152,19 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] Subquery #6 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt index f77919ad43..91cb11a0cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt @@ -22,12 +22,12 @@ : : : +- CometBroadcastHashJoin (53) : : : :- CometBroadcastHashJoin (47) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (46) : : : : +- CometProject (45) : : : : +- CometBroadcastHashJoin (44) : : : : :- CometFilter (4) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) @@ -38,25 +38,25 @@ : : : : : :- CometProject (26) : : : : : : +- CometBroadcastHashJoin (25) : : : : : : :- CometFilter (6) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : : +- CometBroadcastExchange (24) : : : : : : +- CometBroadcastHashJoin (23) : : : : : : :- CometFilter (8) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : : +- CometBroadcastExchange (22) : : : : : : +- CometProject (21) : : : : : : +- CometBroadcastHashJoin (20) : : : : : : :- CometProject (15) : : : : : : : +- CometBroadcastHashJoin (14) : : : : : : : :- CometFilter (10) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : : +- CometBroadcastExchange (13) : : : : : : : +- CometFilter (12) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : : +- CometBroadcastExchange (19) : : : : : : +- CometProject (18) : : : : : : +- CometFilter (17) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : : +- ReusedExchange (27) : : : : +- CometBroadcastExchange (41) : : : : +- CometProject (40) @@ -64,18 +64,18 @@ : : : : :- CometProject (37) : : : : : +- CometBroadcastHashJoin (36) : : : : : :- CometFilter (34) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : : +- ReusedExchange (35) : : : : +- ReusedExchange (38) : : : +- CometBroadcastExchange (52) : : : +- CometBroadcastHashJoin (51) : : : :- CometFilter (49) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : : +- ReusedExchange (50) : : +- CometBroadcastExchange (58) : : +- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) : :- * Filter (82) : : +- * HashAggregate (81) : : +- * CometColumnarToRow (80) @@ -88,7 +88,7 @@ : : : +- CometBroadcastHashJoin (72) : : : :- CometBroadcastHashJoin (70) : : : : :- CometFilter (68) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (67) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : : +- ReusedExchange (69) : : : +- ReusedExchange (71) : : +- ReusedExchange (74) @@ -104,7 +104,7 @@ : : +- CometBroadcastHashJoin (88) : : :- CometBroadcastHashJoin (86) : : : :- CometFilter (84) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (83) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (83) : : : +- ReusedExchange (85) : : +- ReusedExchange (87) : +- ReusedExchange (90) @@ -138,7 +138,7 @@ +- ReusedExchange (125) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -150,7 +150,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -161,7 +161,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -184,7 +184,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -220,7 +220,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -296,7 +296,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -363,7 +363,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -395,7 +395,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -451,7 +451,7 @@ Results [6]: [store AS channel#54, i_brand_id#40, i_class_id#41, i_category_id#4 Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sales#55, number_sales#56] Condition : (isnotnull(sales#55) AND (cast(sales#55 as decimal(32,6)) > cast(Subquery scalar-subquery#57, [id=#58] as decimal(32,6)))) -(67) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62] Batched: true Location: InMemoryFileIndex [] @@ -523,7 +523,7 @@ Results [6]: [catalog AS channel#78, i_brand_id#66, i_class_id#67, i_category_id Input [6]: [channel#78, i_brand_id#66, i_class_id#67, i_category_id#68, sales#79, number_sales#80] Condition : (isnotnull(sales#79) AND (cast(sales#79 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#57, [id=#58] as decimal(32,6)))) -(83) CometScan [native_comet] parquet spark_catalog.default.web_sales +(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -790,22 +790,22 @@ Subquery:1 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquer +- CometUnion (153) :- CometProject (141) : +- CometBroadcastHashJoin (140) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (138) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (138) : +- ReusedExchange (139) :- CometProject (148) : +- CometBroadcastHashJoin (147) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (142) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (142) : +- CometBroadcastExchange (146) : +- CometProject (145) : +- CometFilter (144) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (143) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) +- CometProject (152) +- CometBroadcastHashJoin (151) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (149) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (149) +- ReusedExchange (150) -(138) CometScan [native_comet] parquet spark_catalog.default.store_sales +(138) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#207, ss_list_price#208, ss_sold_date_sk#209] Batched: true Location: InMemoryFileIndex [] @@ -824,14 +824,14 @@ Arguments: [ss_sold_date_sk#209], [d_date_sk#211], Inner, BuildRight Input [4]: [ss_quantity#207, ss_list_price#208, ss_sold_date_sk#209, d_date_sk#211] Arguments: [quantity#212, list_price#213], [ss_quantity#207 AS quantity#212, ss_list_price#208 AS list_price#213] -(142) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(142) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#214, cs_list_price#215, cs_sold_date_sk#216] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#216), dynamicpruningexpression(cs_sold_date_sk#216 IN dynamicpruning#217)] ReadSchema: struct -(143) CometScan [native_comet] parquet spark_catalog.default.date_dim +(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#218, d_year#219] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -859,7 +859,7 @@ Arguments: [cs_sold_date_sk#216], [d_date_sk#218], Inner, BuildRight Input [4]: [cs_quantity#214, cs_list_price#215, cs_sold_date_sk#216, d_date_sk#218] Arguments: [quantity#220, list_price#221], [cs_quantity#214 AS quantity#220, cs_list_price#215 AS list_price#221] -(149) CometScan [native_comet] parquet spark_catalog.default.web_sales +(149) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#222, ws_list_price#223, ws_sold_date_sk#224] Batched: true Location: InMemoryFileIndex [] @@ -914,10 +914,10 @@ BroadcastExchange (163) +- * CometColumnarToRow (162) +- CometProject (161) +- CometFilter (160) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (159) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (159) -(159) CometScan [native_comet] parquet spark_catalog.default.date_dim +(159) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#218, d_year#219] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -946,10 +946,10 @@ BroadcastExchange (168) +- * CometColumnarToRow (167) +- CometProject (166) +- CometFilter (165) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (164) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (164) -(164) CometScan [native_comet] parquet spark_catalog.default.date_dim +(164) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -976,10 +976,10 @@ BroadcastExchange (173) +- * CometColumnarToRow (172) +- CometProject (171) +- CometFilter (170) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (169) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (169) -(169) CometScan [native_comet] parquet spark_catalog.default.date_dim +(169) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#235] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt index 7a46139f98..fa82964f46 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt @@ -32,12 +32,12 @@ WholeStageCodegen (46) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #12 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #17 WholeStageCodegen (1) @@ -45,14 +45,14 @@ WholeStageCodegen (46) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #4 ReusedExchange [d_date_sk] #18 HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] @@ -69,7 +69,7 @@ WholeStageCodegen (46) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -77,12 +77,12 @@ WholeStageCodegen (46) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #5 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #6 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -93,7 +93,7 @@ WholeStageCodegen (46) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) @@ -101,26 +101,26 @@ WholeStageCodegen (46) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 CometProject [i_brand_id,i_class_id,i_category_id] @@ -128,19 +128,19 @@ WholeStageCodegen (46) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #5 CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] WholeStageCodegen (4) Filter [sales] ReusedSubquery [average_sales] #3 @@ -158,7 +158,7 @@ WholeStageCodegen (46) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #5 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 @@ -180,7 +180,7 @@ WholeStageCodegen (46) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #5 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt index e970383aec..4b608ded90 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt @@ -19,30 +19,30 @@ TakeOrderedAndProject (155) : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.item (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) :- * HashAggregate (70) : +- * CometColumnarToRow (69) : +- CometColumnarExchange (68) @@ -61,14 +61,14 @@ TakeOrderedAndProject (155) : : : : : :- CometProject (47) : : : : : : +- CometBroadcastHashJoin (46) : : : : : : :- CometFilter (44) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (43) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (43) : : : : : : +- ReusedExchange (45) : : : : : +- ReusedExchange (48) : : : : +- ReusedExchange (51) : : : +- CometBroadcastExchange (57) : : : +- CometProject (56) : : : +- CometFilter (55) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (54) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (54) : : +- ReusedExchange (60) : +- ReusedExchange (63) :- * HashAggregate (98) @@ -89,14 +89,14 @@ TakeOrderedAndProject (155) : : : : : :- CometProject (75) : : : : : : +- CometBroadcastHashJoin (74) : : : : : : :- CometFilter (72) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (71) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (71) : : : : : : +- ReusedExchange (73) : : : : : +- ReusedExchange (76) : : : : +- ReusedExchange (79) : : : +- CometBroadcastExchange (85) : : : +- CometProject (84) : : : +- CometFilter (83) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (82) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (82) : : +- ReusedExchange (88) : +- ReusedExchange (91) :- * HashAggregate (126) @@ -117,14 +117,14 @@ TakeOrderedAndProject (155) : : : : : :- CometProject (103) : : : : : : +- CometBroadcastHashJoin (102) : : : : : : :- CometFilter (100) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (99) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (99) : : : : : : +- ReusedExchange (101) : : : : : +- ReusedExchange (104) : : : : +- ReusedExchange (107) : : : +- CometBroadcastExchange (113) : : : +- CometProject (112) : : : +- CometFilter (111) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (110) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (110) : : +- ReusedExchange (116) : +- ReusedExchange (119) +- * HashAggregate (153) @@ -145,7 +145,7 @@ TakeOrderedAndProject (155) : : : : :- CometProject (131) : : : : : +- CometBroadcastHashJoin (130) : : : : : :- CometFilter (128) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (127) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (127) : : : : : +- ReusedExchange (129) : : : : +- ReusedExchange (132) : : : +- ReusedExchange (135) @@ -153,10 +153,10 @@ TakeOrderedAndProject (155) : +- ReusedExchange (141) +- CometBroadcastExchange (146) +- CometFilter (145) - +- CometScan [native_comet] parquet spark_catalog.default.item (144) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (144) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -168,7 +168,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -148,10 +148,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt index e6ea677e1e..b49ae70df5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt @@ -22,7 +22,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,12 +30,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt index 2b7a72651a..1f6f727512 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt @@ -13,21 +13,21 @@ TakeOrderedAndProject (26) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) +- BroadcastExchange (18) +- * CometColumnarToRow (17) - +- CometScan [native_comet] parquet spark_catalog.default.warehouse (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -67,7 +67,7 @@ Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_pro (15) CometColumnarToRow [codegen id : 2] Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(16) CometScan [native_comet] parquet spark_catalog.default.warehouse +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output: [] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt index 08de1f27fb..221c6063ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] CometProject [inv_item_sk,inv_quantity_on_hand] CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -24,18 +24,18 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter - CometScan [native_comet] parquet spark_catalog.default.warehouse + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt index 70b413aea3..000792f565 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt @@ -15,18 +15,18 @@ TakeOrderedAndProject (56) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) :- * HashAggregate (33) : +- * CometColumnarToRow (32) : +- CometColumnarExchange (31) @@ -57,7 +57,7 @@ TakeOrderedAndProject (56) +- ReusedExchange (48) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -69,7 +69,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt index 420385984c..9ffdb57abc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt index 5c1d71cef5..c8f5d42144 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt @@ -15,40 +15,40 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (13) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) : : : +- ReusedExchange (14) : : +- BroadcastExchange (25) : : +- * CometColumnarToRow (24) : : +- CometProject (23) : : +- CometBroadcastHashJoin (22) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (20) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) : : +- ReusedExchange (21) : +- BroadcastExchange (33) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,14 +59,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#22] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt index 646bb805ec..fea6a32741 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt index 2d36703c9c..d329f317e9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt @@ -13,36 +13,36 @@ TakeOrderedAndProject (41) : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (12) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) +- CometBroadcastExchange (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -53,14 +53,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -97,7 +97,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan [native_comet] parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -116,7 +116,7 @@ Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] Arguments: [customsk#14], [ws_bill_customer_sk#10 AS customsk#14] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -152,7 +152,7 @@ Arguments: [c_customer_sk#1], [customsk#14], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_comet] parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#20, ca_state#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -180,7 +180,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20, ca_state#22] Arguments: [c_current_cdemo_sk#2, ca_state#22], [c_current_cdemo_sk#2, ca_state#22] -(30) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -243,10 +243,10 @@ BroadcastExchange (46) +- * CometColumnarToRow (45) +- CometProject (44) +- CometFilter (43) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) -(42) CometScan [native_comet] parquet spark_catalog.default.date_dim +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt index 3fd4e91d84..7024842148 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt @@ -16,11 +16,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -28,28 +28,28 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [customsk] #5 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt index 76391e8da4..78fb7563cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt @@ -21,19 +21,19 @@ TakeOrderedAndProject (49) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.store (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) :- * HashAggregate (32) : +- * CometColumnarToRow (31) : +- CometColumnarExchange (30) @@ -50,7 +50,7 @@ TakeOrderedAndProject (49) +- ReusedExchange (33) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -62,7 +62,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -225,7 +225,7 @@ Arguments: hashpartitioning(item_sk#11, d_date#6, 5), ENSURE_REQUIREMENTS, Comet Input [3]: [item_sk#11, d_date#6, cume_sales#24] Arguments: [item_sk#11, d_date#6, cume_sales#24], [item_sk#11 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] -(33) CometScan [native_comet] parquet spark_catalog.default.store_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#25, ss_sales_price#26, ss_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] @@ -442,10 +442,10 @@ BroadcastExchange (85) +- * CometColumnarToRow (84) +- CometProject (83) +- CometFilter (82) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (81) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (81) -(81) CometScan [native_comet] parquet spark_catalog.default.date_dim +(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt index a98de28b8e..8e3c62a654 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt @@ -46,7 +46,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #6 WholeStageCodegen (1) @@ -54,11 +54,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter BroadcastExchange #8 WholeStageCodegen (7) @@ -101,7 +101,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #7 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt index c500003bb8..458b40ef3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt @@ -25,16 +25,16 @@ TakeOrderedAndProject (51) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (40) : +- * Project (39) : +- Window (38) @@ -52,7 +52,7 @@ TakeOrderedAndProject (51) +- ReusedExchange (43) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -67,7 +67,7 @@ Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCode Input [3]: [i_item_sk#1, i_brand#2, i_category#3] Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -92,7 +92,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -116,7 +116,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_comet] parquet spark_catalog.default.call_center +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -283,10 +283,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (55) +- * CometColumnarToRow (54) +- CometFilter (53) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (52) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) -(52) CometScan [native_comet] parquet spark_catalog.default.date_dim +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt index 3986a19dda..a4555a60d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt @@ -34,23 +34,23 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt index 7921bd5e57..45ee60697f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt @@ -22,18 +22,18 @@ : : : :- CometUnion (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) : :- * HashAggregate (45) : : +- * CometColumnarToRow (44) : : +- CometColumnarExchange (43) @@ -46,15 +46,15 @@ : : : :- CometUnion (31) : : : : :- CometProject (27) : : : : : +- CometFilter (26) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (25) : : : : +- CometProject (30) : : : : +- CometFilter (29) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (28) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (28) : : : +- ReusedExchange (32) : : +- CometBroadcastExchange (38) : : +- CometProject (37) : : +- CometFilter (36) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (35) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (35) : +- * HashAggregate (70) : +- * CometColumnarToRow (69) : +- CometColumnarExchange (68) @@ -67,19 +67,19 @@ : : :- CometUnion (56) : : : :- CometProject (48) : : : : +- CometFilter (47) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (46) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (46) : : : +- CometProject (55) : : : +- CometBroadcastHashJoin (54) : : : :- CometBroadcastExchange (50) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (49) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (49) : : : +- CometProject (53) : : : +- CometFilter (52) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (51) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (51) : : +- ReusedExchange (57) : +- CometBroadcastExchange (63) : +- CometProject (62) : +- CometFilter (61) - : +- CometScan [native_comet] parquet spark_catalog.default.web_site (60) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (60) :- * HashAggregate (82) : +- * CometColumnarToRow (81) : +- CometColumnarExchange (80) @@ -96,7 +96,7 @@ +- ReusedExchange (83) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -112,7 +112,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -132,7 +132,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_comet] parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -160,7 +160,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_comet] parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -212,7 +212,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt# Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#35, sum(UnscaledValue(return_amt#10))#36, sum(UnscaledValue(profit#9))#37, sum(UnscaledValue(net_loss#11))#38] Results [5]: [store channel AS channel#39, concat(store, s_store_id#26) AS id#40, MakeDecimal(sum(UnscaledValue(sales_price#8))#35,17,2) AS sales#41, MakeDecimal(sum(UnscaledValue(return_amt#10))#36,17,2) AS returns#42, (MakeDecimal(sum(UnscaledValue(profit#9))#37,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#38,17,2)) AS profit#43] -(25) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#44, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -228,7 +228,7 @@ Condition : isnotnull(cs_catalog_page_sk#44) Input [4]: [cs_catalog_page_sk#44, cs_ext_sales_price#45, cs_net_profit#46, cs_sold_date_sk#47] Arguments: [page_sk#49, date_sk#50, sales_price#51, profit#52, return_amt#53, net_loss#54], [cs_catalog_page_sk#44 AS page_sk#49, cs_sold_date_sk#47 AS date_sk#50, cs_ext_sales_price#45 AS sales_price#51, cs_net_profit#46 AS profit#52, 0.00 AS return_amt#53, 0.00 AS net_loss#54] -(28) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#55, cr_return_amount#56, cr_net_loss#57, cr_returned_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -260,7 +260,7 @@ Arguments: [date_sk#50], [d_date_sk#65], Inner, BuildRight Input [7]: [page_sk#49, date_sk#50, sales_price#51, profit#52, return_amt#53, net_loss#54, d_date_sk#65] Arguments: [page_sk#49, sales_price#51, profit#52, return_amt#53, net_loss#54], [page_sk#49, sales_price#51, profit#52, return_amt#53, net_loss#54] -(35) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#66, cp_catalog_page_id#67] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -312,7 +312,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#51)), sum(UnscaledValue(return_amt Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#51))#77, sum(UnscaledValue(return_amt#53))#78, sum(UnscaledValue(profit#52))#79, sum(UnscaledValue(net_loss#54))#80] Results [5]: [catalog channel AS channel#81, concat(catalog_page, cp_catalog_page_id#68) AS id#82, MakeDecimal(sum(UnscaledValue(sales_price#51))#77,17,2) AS sales#83, MakeDecimal(sum(UnscaledValue(return_amt#53))#78,17,2) AS returns#84, (MakeDecimal(sum(UnscaledValue(profit#52))#79,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#54))#80,17,2)) AS profit#85] -(46) CometScan [native_comet] parquet spark_catalog.default.web_sales +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#86, ws_ext_sales_price#87, ws_net_profit#88, ws_sold_date_sk#89] Batched: true Location: InMemoryFileIndex [] @@ -328,7 +328,7 @@ Condition : isnotnull(ws_web_site_sk#86) Input [4]: [ws_web_site_sk#86, ws_ext_sales_price#87, ws_net_profit#88, ws_sold_date_sk#89] Arguments: [wsr_web_site_sk#91, date_sk#92, sales_price#93, profit#94, return_amt#95, net_loss#96], [ws_web_site_sk#86 AS wsr_web_site_sk#91, ws_sold_date_sk#89 AS date_sk#92, ws_ext_sales_price#87 AS sales_price#93, ws_net_profit#88 AS profit#94, 0.00 AS return_amt#95, 0.00 AS net_loss#96] -(49) CometScan [native_comet] parquet spark_catalog.default.web_returns +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#97, wr_order_number#98, wr_return_amt#99, wr_net_loss#100, wr_returned_date_sk#101] Batched: true Location: InMemoryFileIndex [] @@ -339,7 +339,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -111,7 +111,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 5] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_comet] parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -125,7 +125,7 @@ Condition : isnotnull(s_store_sk#7) (12) CometColumnarToRow [codegen id : 4] Input [3]: [s_store_sk#7, s_county#8, s_state#9] -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -386,10 +386,10 @@ BroadcastExchange (72) +- * CometColumnarToRow (71) +- CometProject (70) +- CometFilter (69) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) -(68) CometScan [native_comet] parquet spark_catalog.default.date_dim +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt index d1ad775911..b6de40bbcf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,11 +36,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -49,7 +49,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (3) @@ -73,12 +73,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_net_profit,ss_sold_date_sk,s_state] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [s_store_sk,s_state] #9 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #5 WholeStageCodegen (13) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt index 5b44e8488c..707d448aff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt @@ -27,45 +27,45 @@ : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (3) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (8) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : : : : : : : +- CometBroadcastExchange (15) : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (13) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) : : : : : : +- CometBroadcastExchange (21) : : : : : : +- CometProject (20) : : : : : : +- CometFilter (19) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (18) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) : : : : : +- CometBroadcastExchange (27) : : : : : +- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) : : : : +- CometBroadcastExchange (33) : : : : +- CometProject (32) : : : : +- CometFilter (31) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : : : +- CometBroadcastExchange (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) : : +- CometBroadcastExchange (43) : : +- CometFilter (42) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) : +- CometBroadcastExchange (48) : +- CometFilter (47) - : +- CometScan [native_comet] parquet spark_catalog.default.promotion (46) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) +- CometSort (57) +- CometExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (53) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -77,7 +77,7 @@ ReadSchema: struct date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_comet] parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_comet] parquet spark_catalog.default.date_dim +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt index a89be99fc8..c770348a8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt index a96653ed38..540d9d6ff1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt @@ -17,13 +17,13 @@ TakeOrderedAndProject (79) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- BroadcastExchange (38) : : +- * HashAggregate (37) : : +- * CometColumnarToRow (36) @@ -36,13 +36,13 @@ TakeOrderedAndProject (79) : : : +- CometBroadcastHashJoin (26) : : : :- CometProject (22) : : : : +- CometFilter (21) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (25) : : : +- CometFilter (24) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : +- CometBroadcastExchange (30) : : +- CometFilter (29) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (28) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (28) : +- BroadcastExchange (57) : +- * Filter (56) : +- * HashAggregate (55) @@ -56,10 +56,10 @@ TakeOrderedAndProject (79) : : +- CometBroadcastHashJoin (46) : : :- CometProject (42) : : : +- CometFilter (41) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (40) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (40) : : +- CometBroadcastExchange (45) : : +- CometFilter (44) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (43) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) : +- ReusedExchange (48) +- BroadcastExchange (76) +- * HashAggregate (75) @@ -73,14 +73,14 @@ TakeOrderedAndProject (79) : +- CometBroadcastHashJoin (66) : :- CometProject (62) : : +- CometFilter (61) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (60) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (60) : +- CometBroadcastExchange (65) : +- CometFilter (64) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (63) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (63) +- ReusedExchange (68) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -95,7 +95,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharC Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -120,7 +120,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -172,7 +172,7 @@ Results [2]: [c_customer_id#5 AS customer_id#17, MakeDecimal(sum(UnscaledValue(s Input [2]: [customer_id#17, year_total#18] Condition : (isnotnull(year_total#18) AND (year_total#18 > 0.00)) -(20) CometScan [native_comet] parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -187,7 +187,7 @@ Condition : (isnotnull(c_customer_sk#19) AND isnotnull(static_invoke(CharVarchar Input [4]: [c_customer_sk#19, c_customer_id#20, c_first_name#21, c_last_name#22] Arguments: [c_customer_sk#19, c_customer_id#23, c_first_name#24, c_last_name#25], [c_customer_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#20, 16)) AS c_customer_id#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#21, 20)) AS c_first_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#22, 30)) AS c_last_name#25] -(23) CometScan [native_comet] parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#26, ss_net_paid#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -212,7 +212,7 @@ Arguments: [c_customer_sk#19], [ss_customer_sk#26], Inner, BuildRight Input [7]: [c_customer_sk#19, c_customer_id#23, c_first_name#24, c_last_name#25, ss_customer_sk#26, ss_net_paid#27, ss_sold_date_sk#28] Arguments: [c_customer_id#23, c_first_name#24, c_last_name#25, ss_net_paid#27, ss_sold_date_sk#28], [c_customer_id#23, c_first_name#24, c_last_name#25, ss_net_paid#27, ss_sold_date_sk#28] -(28) CometScan [native_comet] parquet spark_catalog.default.date_dim +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#30, d_year#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -270,7 +270,7 @@ Right keys [1]: [customer_id#34] Join type: Inner Join condition: None -(40) CometScan [native_comet] parquet spark_catalog.default.customer +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -285,7 +285,7 @@ Condition : (isnotnull(c_customer_sk#38) AND isnotnull(static_invoke(CharVarchar Input [4]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41] Arguments: [c_customer_sk#38, c_customer_id#42, c_first_name#43, c_last_name#44], [c_customer_sk#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#39, 16)) AS c_customer_id#42, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#40, 20)) AS c_first_name#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#41, 30)) AS c_last_name#44] -(43) CometScan [native_comet] parquet spark_catalog.default.web_sales +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#45, ws_net_paid#46, ws_sold_date_sk#47] Batched: true Location: InMemoryFileIndex [] @@ -364,7 +364,7 @@ Join condition: None Output [7]: [customer_id#17, year_total#18, customer_id#34, customer_first_name#35, customer_last_name#36, year_total#37, year_total#55] Input [8]: [customer_id#17, year_total#18, customer_id#34, customer_first_name#35, customer_last_name#36, year_total#37, customer_id#54, year_total#55] -(60) CometScan [native_comet] parquet spark_catalog.default.customer +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#56, c_customer_id#57, c_first_name#58, c_last_name#59] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -379,7 +379,7 @@ Condition : (isnotnull(c_customer_sk#56) AND isnotnull(static_invoke(CharVarchar Input [4]: [c_customer_sk#56, c_customer_id#57, c_first_name#58, c_last_name#59] Arguments: [c_customer_sk#56, c_customer_id#60, c_first_name#61, c_last_name#62], [c_customer_sk#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#57, 16)) AS c_customer_id#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#58, 20)) AS c_first_name#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#59, 30)) AS c_last_name#62] -(63) CometScan [native_comet] parquet spark_catalog.default.web_sales +(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#63, ws_net_paid#64, ws_sold_date_sk#65] Batched: true Location: InMemoryFileIndex [] @@ -464,10 +464,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (83) +- * CometColumnarToRow (82) +- CometFilter (81) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (80) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (80) -(80) CometScan [native_comet] parquet spark_catalog.default.date_dim +(80) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -489,10 +489,10 @@ Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#28 IN d BroadcastExchange (87) +- * CometColumnarToRow (86) +- CometFilter (85) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (84) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (84) -(84) CometScan [native_comet] parquet spark_catalog.default.date_dim +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#30, d_year#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt index 1e55dff056..a4e489b3e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt @@ -20,20 +20,20 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -51,20 +51,20 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) @@ -83,10 +83,10 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 InputAdapter @@ -106,9 +106,9 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt index fa9589797a..66aad99aa1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt @@ -24,19 +24,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -46,14 +46,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -63,14 +63,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (124) +- CometColumnarExchange (123) +- * Filter (122) @@ -92,11 +92,11 @@ : : :- CometProject (73) : : : +- CometBroadcastHashJoin (72) : : : :- CometFilter (70) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (69) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (69) : : : +- ReusedExchange (71) : : +- CometBroadcastExchange (76) : : +- CometFilter (75) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (74) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) : +- CometSort (82) : +- ReusedExchange (81) :- CometProject (98) @@ -108,7 +108,7 @@ : : :- CometProject (89) : : : +- CometBroadcastHashJoin (88) : : : :- CometFilter (86) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (85) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (85) : : : +- ReusedExchange (87) : : +- ReusedExchange (90) : +- CometSort (96) @@ -122,14 +122,14 @@ : :- CometProject (103) : : +- CometBroadcastHashJoin (102) : : :- CometFilter (100) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (99) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (99) : : +- ReusedExchange (101) : +- ReusedExchange (104) +- CometSort (110) +- ReusedExchange (109) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -141,7 +141,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -84,7 +84,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -252,10 +252,10 @@ BroadcastExchange (48) +- * CometColumnarToRow (47) +- CometProject (46) +- CometFilter (45) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (44) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) -(44) CometScan [native_comet] parquet spark_catalog.default.date_dim +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt index 292561efd8..38f1454146 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,15 +36,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #6 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] WholeStageCodegen (5) HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] CometColumnarToRow diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt index bc67841345..adc2199890 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt @@ -17,18 +17,18 @@ :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -40,7 +40,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -68,7 +68,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -157,10 +157,10 @@ BroadcastExchange (32) +- * CometColumnarToRow (31) +- CometProject (30) +- CometFilter (29) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (28) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (28) -(28) CometScan [native_comet] parquet spark_catalog.default.date_dim +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt index c2d6c3a4e9..6db0820117 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt @@ -26,7 +26,7 @@ WholeStageCodegen (5) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -34,12 +34,12 @@ WholeStageCodegen (5) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt index e6ce5e728d..ac9e39c190 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt @@ -12,36 +12,36 @@ : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (12) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) +- CometBroadcastExchange (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -52,14 +52,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -96,7 +96,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan [native_comet] parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ Arguments: [ws_sold_date_sk#11], [d_date_sk#7], Inner, BuildRight Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#7] Arguments: [customer_sk#13], [ws_bill_customer_sk#10 AS customer_sk#13] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customer_sk#13], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_comet] parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#19, ca_county#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#19], Inner, BuildRight Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#19] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(30) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_education_status#24, cd_purchase_estimate#25, cd_credit_rating#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt index e6c4ab5862..08c0ad906c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt @@ -13,11 +13,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -25,28 +25,28 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [customer_sk] #5 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometProject [cs_ship_customer_sk] [customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt index 1d9bfdb401..49c1c8e0d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,10 +51,10 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) : +- ReusedExchange (44) +- CometBroadcastExchange (68) +- CometHashAggregate (67) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -88,7 +88,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16 Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -170,7 +170,7 @@ Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Batched: true Location: InMemoryFileIndex [] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#23, year_total#24] Right output [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -257,7 +257,7 @@ Condition : (isnotnull(c_customer_sk#46) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] Arguments: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59], [c_customer_sk#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#47, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#48, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#49, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#50, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#52, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#53, 50, true, false, true) AS c_email_address#59] -(39) CometScan [native_comet] parquet spark_catalog.default.web_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] @@ -325,7 +325,7 @@ Arguments: [customer_id#23], [customer_id#68], Inner, BuildRight Input [9]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, customer_id#68, year_total#69] Arguments: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69], [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -340,7 +340,7 @@ Condition : (isnotnull(c_customer_sk#70) AND isnotnull(staticinvoke(class org.ap Input [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Arguments: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59], [c_customer_sk#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#71, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#72, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#73, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#74, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#75, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#76, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#77, 50, true, false, true) AS c_email_address#59] -(57) CometScan [native_comet] parquet spark_catalog.default.web_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] Batched: true Location: InMemoryFileIndex [] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#38, d_year#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt index 1b7dc9294a..9d22d7d913 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt @@ -17,20 +17,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 @@ -41,20 +41,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] @@ -66,10 +66,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -82,9 +82,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt index 77d08a829f..410aecff1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt @@ -13,18 +13,18 @@ TakeOrderedAndProject (23) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +36,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt index 2d88b65482..ad8631ea1d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt index 48b0d32634..33224acffb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt @@ -12,12 +12,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -28,25 +28,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -54,18 +54,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -77,16 +77,16 @@ : +- CometBroadcastHashJoin (70) : :- CometBroadcastHashJoin (68) : : :- CometFilter (66) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) : : +- ReusedExchange (67) : +- ReusedExchange (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -109,7 +109,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -132,7 +132,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -244,7 +244,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -389,7 +389,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) -(65) CometScan [native_comet] parquet spark_catalog.default.store_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] Batched: true Location: InMemoryFileIndex [] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#55], [i_item_sk#60], Inner, BuildRight Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] -(72) CometScan [native_comet] parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -493,19 +493,19 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (98) :- CometProject (89) : +- CometBroadcastHashJoin (88) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (86) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) : +- ReusedExchange (87) :- CometProject (93) : +- CometBroadcastHashJoin (92) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (90) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) : +- ReusedExchange (91) +- CometProject (97) +- CometBroadcastHashJoin (96) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (94) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) +- ReusedExchange (95) -(86) CometScan [native_comet] parquet spark_catalog.default.store_sales +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76] Batched: true Location: InMemoryFileIndex [] @@ -524,7 +524,7 @@ Arguments: [ss_sold_date_sk#76], [d_date_sk#78], Inner, BuildRight Input [4]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76, d_date_sk#78] Arguments: [quantity#79, list_price#80], [ss_quantity#74 AS quantity#79, ss_list_price#75 AS list_price#80] -(90) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] @@ -543,7 +543,7 @@ Arguments: [cs_sold_date_sk#83], [d_date_sk#85], Inner, BuildRight Input [4]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83, d_date_sk#85] Arguments: [quantity#86, list_price#87], [cs_quantity#81 AS quantity#86, cs_list_price#82 AS list_price#87] -(94) CometScan [native_comet] parquet spark_catalog.default.web_sales +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90] Batched: true Location: InMemoryFileIndex [] @@ -595,10 +595,10 @@ BroadcastExchange (107) +- * CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (103) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(103) CometScan [native_comet] parquet spark_catalog.default.date_dim +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -624,10 +624,10 @@ Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subque * CometColumnarToRow (111) +- CometProject (110) +- CometFilter (109) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) -(108) CometScan [native_comet] parquet spark_catalog.default.date_dim +(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -650,10 +650,10 @@ BroadcastExchange (116) +- * CometColumnarToRow (115) +- CometProject (114) +- CometFilter (113) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (112) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) -(112) CometScan [native_comet] parquet spark_catalog.default.date_dim +(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#99] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -688,10 +688,10 @@ BroadcastExchange (121) +- * CometColumnarToRow (120) +- CometProject (119) +- CometFilter (118) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (117) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) -(117) CometScan [native_comet] parquet spark_catalog.default.date_dim +(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#64, d_week_seq#65] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -717,10 +717,10 @@ Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subqu * CometColumnarToRow (125) +- CometProject (124) +- CometFilter (123) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (122) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) -(122) CometScan [native_comet] parquet spark_catalog.default.date_dim +(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt index 0e0b2ea4b8..5d5557bc59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt @@ -14,17 +14,17 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -50,13 +50,13 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -67,7 +67,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #3 BroadcastExchange #6 WholeStageCodegen (1) @@ -75,26 +75,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #3 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -102,20 +102,20 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 @@ -128,7 +128,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #5 BroadcastExchange #17 WholeStageCodegen (1) @@ -142,12 +142,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] ReusedSubquery [d_week_seq] #6 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt index c961c0088e..4ac6c99894 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt @@ -19,12 +19,12 @@ : : : +- CometBroadcastHashJoin (53) : : : :- CometBroadcastHashJoin (47) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (46) : : : : +- CometProject (45) : : : : +- CometBroadcastHashJoin (44) : : : : :- CometFilter (4) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) @@ -35,25 +35,25 @@ : : : : : :- CometProject (26) : : : : : : +- CometBroadcastHashJoin (25) : : : : : : :- CometFilter (6) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (5) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : : +- CometBroadcastExchange (24) : : : : : : +- CometBroadcastHashJoin (23) : : : : : : :- CometFilter (8) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (7) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : : +- CometBroadcastExchange (22) : : : : : : +- CometProject (21) : : : : : : +- CometBroadcastHashJoin (20) : : : : : : :- CometProject (15) : : : : : : : +- CometBroadcastHashJoin (14) : : : : : : : :- CometFilter (10) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : : +- CometBroadcastExchange (13) : : : : : : : +- CometFilter (12) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (11) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : : +- CometBroadcastExchange (19) : : : : : : +- CometProject (18) : : : : : : +- CometFilter (17) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (16) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : : +- ReusedExchange (27) : : : : +- CometBroadcastExchange (41) : : : : +- CometProject (40) @@ -61,18 +61,18 @@ : : : : :- CometProject (37) : : : : : +- CometBroadcastHashJoin (36) : : : : : :- CometFilter (34) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (33) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : : +- ReusedExchange (35) : : : : +- ReusedExchange (38) : : : +- CometBroadcastExchange (52) : : : +- CometBroadcastHashJoin (51) : : : :- CometFilter (49) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : : +- ReusedExchange (50) : : +- CometBroadcastExchange (58) : : +- CometProject (57) : : +- CometFilter (56) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (55) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) : :- CometFilter (78) : : +- CometHashAggregate (77) : : +- CometExchange (76) @@ -83,7 +83,7 @@ : : : +- CometBroadcastHashJoin (70) : : : :- CometBroadcastHashJoin (68) : : : : :- CometFilter (66) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (65) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (65) : : : : +- ReusedExchange (67) : : : +- ReusedExchange (69) : : +- ReusedExchange (72) @@ -97,7 +97,7 @@ : : +- CometBroadcastHashJoin (84) : : :- CometBroadcastHashJoin (82) : : : :- CometFilter (80) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (79) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (79) : : : +- ReusedExchange (81) : : +- ReusedExchange (83) : +- ReusedExchange (86) @@ -123,7 +123,7 @@ +- ReusedExchange (112) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -135,7 +135,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -146,7 +146,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] Condition : isnotnull(ss_item_sk#10) -(7) CometScan [native_comet] parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -169,7 +169,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] Condition : isnotnull(cs_item_sk#17) -(11) CometScan [native_comet] parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -205,7 +205,7 @@ Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -(16) CometScan [native_comet] parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -281,7 +281,7 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(33) CometScan [native_comet] parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -348,7 +348,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#38] Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight -(48) CometScan [native_comet] parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -380,7 +380,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -(55) CometScan [native_comet] parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -426,7 +426,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) -(65) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] Batched: true Location: InMemoryFileIndex [] @@ -488,7 +488,7 @@ Functions [2]: [sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), Input [6]: [channel#67, i_brand_id#60, i_class_id#61, i_category_id#62, sales#68, number_sales#69] Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) -(79) CometScan [native_comet] parquet spark_catalog.default.web_sales +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#70, ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73] Batched: true Location: InMemoryFileIndex [] @@ -695,22 +695,22 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer +- CometUnion (138) :- CometProject (126) : +- CometBroadcastHashJoin (125) - : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (123) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (123) : +- ReusedExchange (124) :- CometProject (133) : +- CometBroadcastHashJoin (132) - : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (127) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (127) : +- CometBroadcastExchange (131) : +- CometProject (130) : +- CometFilter (129) - : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (128) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (128) +- CometProject (137) +- CometBroadcastHashJoin (136) - :- CometScan [native_comet] parquet spark_catalog.default.web_sales (134) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (134) +- ReusedExchange (135) -(123) CometScan [native_comet] parquet spark_catalog.default.store_sales +(123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123] Batched: true Location: InMemoryFileIndex [] @@ -729,14 +729,14 @@ Arguments: [ss_sold_date_sk#123], [d_date_sk#125], Inner, BuildRight Input [4]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123, d_date_sk#125] Arguments: [quantity#126, list_price#127], [ss_quantity#121 AS quantity#126, ss_list_price#122 AS list_price#127] -(127) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#130), dynamicpruningexpression(cs_sold_date_sk#130 IN dynamicpruning#131)] ReadSchema: struct -(128) CometScan [native_comet] parquet spark_catalog.default.date_dim +(128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -764,7 +764,7 @@ Arguments: [cs_sold_date_sk#130], [d_date_sk#132], Inner, BuildRight Input [4]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130, d_date_sk#132] Arguments: [quantity#134, list_price#135], [cs_quantity#128 AS quantity#134, cs_list_price#129 AS list_price#135] -(134) CometScan [native_comet] parquet spark_catalog.default.web_sales +(134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] Batched: true Location: InMemoryFileIndex [] @@ -812,10 +812,10 @@ BroadcastExchange (147) +- * CometColumnarToRow (146) +- CometProject (145) +- CometFilter (144) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (143) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) -(143) CometScan [native_comet] parquet spark_catalog.default.date_dim +(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -844,10 +844,10 @@ BroadcastExchange (152) +- * CometColumnarToRow (151) +- CometProject (150) +- CometFilter (149) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (148) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (148) -(148) CometScan [native_comet] parquet spark_catalog.default.date_dim +(148) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -874,10 +874,10 @@ BroadcastExchange (157) +- * CometColumnarToRow (156) +- CometProject (155) +- CometFilter (154) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (153) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (153) -(153) CometScan [native_comet] parquet spark_catalog.default.date_dim +(153) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#146] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt index 7f21febd85..047cf5ecfe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt @@ -21,12 +21,12 @@ WholeStageCodegen (1) CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #12 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #4 BroadcastExchange #17 WholeStageCodegen (1) @@ -34,14 +34,14 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk] #18 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #4 ReusedExchange [d_date_sk] #18 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] @@ -53,7 +53,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -61,12 +61,12 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk] #5 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #6 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -77,7 +77,7 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) @@ -85,26 +85,26 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #2 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 CometProject [i_brand_id,i_class_id,i_category_id] @@ -112,19 +112,19 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 ReusedExchange [d_date_sk] #12 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #5 CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] @@ -136,7 +136,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #5 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 @@ -152,7 +152,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [ss_item_sk] #5 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt index 0a1b71c705..86072fd830 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt @@ -18,30 +18,30 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometFilter (16) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) : : : +- CometBroadcastExchange (23) : : : +- CometProject (22) : : : +- CometFilter (21) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan [native_comet] parquet spark_catalog.default.item (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) :- CometHashAggregate (66) : +- CometExchange (65) : +- CometHashAggregate (64) @@ -58,14 +58,14 @@ : : : : : :- CometProject (45) : : : : : : +- CometBroadcastHashJoin (44) : : : : : : :- CometFilter (42) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (41) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (41) : : : : : : +- ReusedExchange (43) : : : : : +- ReusedExchange (46) : : : : +- ReusedExchange (49) : : : +- CometBroadcastExchange (55) : : : +- CometProject (54) : : : +- CometFilter (53) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (52) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (52) : : +- ReusedExchange (58) : +- ReusedExchange (61) :- CometHashAggregate (92) @@ -84,14 +84,14 @@ : : : : : :- CometProject (71) : : : : : : +- CometBroadcastHashJoin (70) : : : : : : :- CometFilter (68) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (67) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : : : : +- ReusedExchange (69) : : : : : +- ReusedExchange (72) : : : : +- ReusedExchange (75) : : : +- CometBroadcastExchange (81) : : : +- CometProject (80) : : : +- CometFilter (79) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (78) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (78) : : +- ReusedExchange (84) : +- ReusedExchange (87) :- CometHashAggregate (118) @@ -110,14 +110,14 @@ : : : : : :- CometProject (97) : : : : : : +- CometBroadcastHashJoin (96) : : : : : : :- CometFilter (94) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (93) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (93) : : : : : : +- ReusedExchange (95) : : : : : +- ReusedExchange (98) : : : : +- ReusedExchange (101) : : : +- CometBroadcastExchange (107) : : : +- CometProject (106) : : : +- CometFilter (105) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (104) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (104) : : +- ReusedExchange (110) : +- ReusedExchange (113) +- CometHashAggregate (143) @@ -136,7 +136,7 @@ : : : : :- CometProject (123) : : : : : +- CometBroadcastHashJoin (122) : : : : : :- CometFilter (120) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (119) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (119) : : : : : +- ReusedExchange (121) : : : : +- ReusedExchange (124) : : : +- ReusedExchange (127) @@ -144,10 +144,10 @@ : +- ReusedExchange (133) +- CometBroadcastExchange (138) +- CometFilter (137) - +- CometScan [native_comet] parquet spark_catalog.default.item (136) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (136) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -159,7 +159,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -136,10 +136,10 @@ BroadcastExchange (28) +- * CometColumnarToRow (27) +- CometProject (26) +- CometFilter (25) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) -(24) CometScan [native_comet] parquet spark_catalog.default.date_dim +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt index 75319785fa..51d992e9cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,12 +24,12 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt index a480e94ee0..f85c898208 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt @@ -13,21 +13,21 @@ TakeOrderedAndProject (26) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) +- BroadcastExchange (18) +- * CometColumnarToRow (17) - +- CometScan [native_comet] parquet spark_catalog.default.warehouse (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -39,7 +39,7 @@ ReadSchema: struct Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -67,7 +67,7 @@ Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -98,7 +98,7 @@ Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_pro (15) CometColumnarToRow [codegen id : 2] Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(16) CometScan [native_comet] parquet spark_catalog.default.warehouse +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output: [] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -155,10 +155,10 @@ BroadcastExchange (31) +- * CometColumnarToRow (30) +- CometProject (29) +- CometFilter (28) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) -(27) CometScan [native_comet] parquet spark_catalog.default.date_dim +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt index 08de1f27fb..221c6063ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt @@ -16,7 +16,7 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] CometProject [inv_item_sk,inv_quantity_on_hand] CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #2 WholeStageCodegen (1) @@ -24,18 +24,18 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter - CometScan [native_comet] parquet spark_catalog.default.warehouse + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt index 743f9cafdf..189a54b584 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt @@ -14,18 +14,18 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) :- CometHashAggregate (29) : +- CometExchange (28) : +- CometHashAggregate (27) @@ -48,7 +48,7 @@ +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ ReadSchema: struct= 15) AND (cnt#17 <= 20)) -(25) CometScan [native_comet] parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -194,10 +194,10 @@ BroadcastExchange (38) +- * CometColumnarToRow (37) +- CometProject (36) +- CometFilter (35) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (34) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) -(34) CometScan [native_comet] parquet spark_catalog.default.date_dim +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_dom#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt index 420385984c..9ffdb57abc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt @@ -16,7 +16,7 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -24,20 +24,20 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] CometBroadcastExchange [hd_demo_sk] #6 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt index 36a81cf831..5df7e6870f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt @@ -15,40 +15,40 @@ TakeOrderedAndProject (47) : : : :- * CometColumnarToRow (12) : : : : +- CometBroadcastHashJoin (11) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (10) : : : : +- CometProject (9) : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- BroadcastExchange (18) : : : +- * CometColumnarToRow (17) : : : +- CometProject (16) : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (13) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) : : : +- ReusedExchange (14) : : +- BroadcastExchange (25) : : +- * CometColumnarToRow (24) : : +- CometProject (23) : : +- CometBroadcastHashJoin (22) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (20) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) : : +- ReusedExchange (21) : +- BroadcastExchange (33) : +- * CometColumnarToRow (32) : +- CometProject (31) : +- CometFilter (30) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) +- BroadcastExchange (40) +- * CometColumnarToRow (39) +- CometProject (38) +- CometFilter (37) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -59,14 +59,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -106,7 +106,7 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight (12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan [native_comet] parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -138,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#12] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -178,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan [native_comet] parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_state#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -210,7 +210,7 @@ Join condition: None Output [2]: [c_current_cdemo_sk#4, ca_state#20] Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18, ca_state#20] -(36) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -274,10 +274,10 @@ BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometProject (50) +- CometFilter (49) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) -(48) CometScan [native_comet] parquet spark_catalog.default.date_dim +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt index 646bb805ec..fea6a32741 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt @@ -18,11 +18,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -30,11 +30,11 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) @@ -42,7 +42,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -52,7 +52,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 InputAdapter @@ -62,7 +62,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #8 WholeStageCodegen (4) @@ -70,4 +70,4 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ InputAdapter CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt index c63e93f4f6..97c7d7659b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt @@ -12,36 +12,36 @@ : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan [native_comet] parquet spark_catalog.default.store_sales (3) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan [native_comet] parquet spark_catalog.default.web_sales (12) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) +- CometBroadcastExchange (33) +- CometProject (32) +- CometFilter (31) - +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -52,14 +52,14 @@ ReadSchema: struct -(4) CometScan [native_comet] parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -96,7 +96,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan [native_comet] parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ Arguments: [ws_sold_date_sk#11], [d_date_sk#7], Inner, BuildRight Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#7] Arguments: [customsk#13], [ws_bill_customer_sk#10 AS customsk#13] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -151,7 +151,7 @@ Arguments: [c_customer_sk#1], [customsk#13], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan [native_comet] parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#19, ca_state#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -179,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#19], Inner, BuildRight Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#19, ca_state#21] Arguments: [c_current_cdemo_sk#2, ca_state#21], [c_current_cdemo_sk#2, ca_state#21] -(30) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -235,10 +235,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt index 6a55cdf307..0aa670f8e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt @@ -13,11 +13,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -25,28 +25,28 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [customsk] #5 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt index a5981fbfb3..0f966424f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt @@ -19,19 +19,19 @@ TakeOrderedAndProject (43) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.item (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.store (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -44,7 +44,7 @@ TakeOrderedAndProject (43) +- ReusedExchange (29) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -56,7 +56,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -211,7 +211,7 @@ Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometC Input [3]: [item_sk#9, d_date#6, cume_sales#22] Arguments: [item_sk#9, d_date#6, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] -(31) CometScan [native_comet] parquet spark_catalog.default.store_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -418,10 +418,10 @@ BroadcastExchange (81) +- * CometColumnarToRow (80) +- CometProject (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt index 1714124cc9..172baee830 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt @@ -40,7 +40,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #6 WholeStageCodegen (1) @@ -48,11 +48,11 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [d_date_sk,d_date] #7 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter BroadcastExchange #8 WholeStageCodegen (3) @@ -89,7 +89,7 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_date] #7 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt index 02e411b04f..d1daee98c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt @@ -23,16 +23,16 @@ TakeOrderedAndProject (48) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.call_center (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) : +- BroadcastExchange (37) : +- * Project (36) : +- Window (35) @@ -49,7 +49,7 @@ TakeOrderedAndProject (48) +- ReusedExchange (40) -(1) CometScan [native_comet] parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -64,7 +64,7 @@ Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache Input [3]: [i_item_sk#1, i_brand#2, i_category#3] Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -89,7 +89,7 @@ Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -113,7 +113,7 @@ Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -(14) CometScan [native_comet] parquet spark_catalog.default.call_center +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#14, cc_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -265,10 +265,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dyn BroadcastExchange (52) +- * CometColumnarToRow (51) +- CometFilter (50) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) -(49) CometScan [native_comet] parquet spark_catalog.default.date_dim +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_moy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt index 9ac1e20eec..7df61c1995 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt @@ -28,23 +28,23 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] CometFilter [i_item_sk,i_brand,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #6 CometFilter [cc_call_center_sk,cc_name] - CometScan [native_comet] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt index e273c6ac21..0ada41413b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt @@ -21,18 +21,18 @@ : : : :- CometUnion (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) : :- * HashAggregate (43) : : +- * CometColumnarToRow (42) : : +- CometExchange (41) @@ -44,15 +44,15 @@ : : : :- CometUnion (30) : : : : :- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) : : : : +- CometProject (29) : : : : +- CometFilter (28) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (27) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) : : : +- ReusedExchange (31) : : +- CometBroadcastExchange (37) : : +- CometProject (36) : : +- CometFilter (35) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_page (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) : +- * HashAggregate (67) : +- * CometColumnarToRow (66) : +- CometExchange (65) @@ -64,19 +64,19 @@ : : :- CometUnion (54) : : : :- CometProject (46) : : : : +- CometFilter (45) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (44) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) : : : +- CometProject (53) : : : +- CometBroadcastHashJoin (52) : : : :- CometBroadcastExchange (48) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (47) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) : : : +- CometProject (51) : : : +- CometFilter (50) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) : : +- ReusedExchange (55) : +- CometBroadcastExchange (61) : +- CometProject (60) : +- CometFilter (59) - : +- CometScan [native_comet] parquet spark_catalog.default.web_site (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) :- * HashAggregate (79) : +- * CometColumnarToRow (78) : +- CometColumnarExchange (77) @@ -93,7 +93,7 @@ +- ReusedExchange (80) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -109,7 +109,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -129,7 +129,7 @@ Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, n Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] -(8) CometScan [native_comet] parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -157,7 +157,7 @@ Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -(14) CometScan [native_comet] parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#24, s_store_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -204,7 +204,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt# Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] Results [5]: [store channel AS channel#35, concat(store, s_store_id#26) AS id#36, MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#39] -(24) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -220,7 +220,7 @@ Condition : isnotnull(cs_catalog_page_sk#40) Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] -(27) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -252,7 +252,7 @@ Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -(34) CometScan [native_comet] parquet spark_catalog.default.catalog_page +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] @@ -299,7 +299,7 @@ Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] Results [5]: [catalog channel AS channel#73, concat(catalog_page, cp_catalog_page_id#64) AS id#74, MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#75, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#76, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#77] -(44) CometScan [native_comet] parquet spark_catalog.default.web_sales +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Batched: true Location: InMemoryFileIndex [] @@ -315,7 +315,7 @@ Condition : isnotnull(ws_web_site_sk#78) Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] -(47) CometScan [native_comet] parquet spark_catalog.default.web_returns +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] Batched: true Location: InMemoryFileIndex [] @@ -326,7 +326,7 @@ ReadSchema: struct Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) -(22) CometScan [native_comet] parquet spark_catalog.default.item +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_current_price#15, i_category#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -234,10 +234,10 @@ BroadcastExchange (45) +- * CometColumnarToRow (44) +- CometProject (43) +- CometFilter (42) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_comet] parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#10, d_month_seq#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -266,10 +266,10 @@ Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquer +- CometHashAggregate (49) +- CometProject (48) +- CometFilter (47) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) -(46) CometScan [native_comet] parquet spark_catalog.default.date_dim +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#11, d_year#24, d_moy#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt index 5c979b2233..9e3765b84c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt @@ -16,13 +16,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -39,18 +39,18 @@ WholeStageCodegen (1) CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] ReusedSubquery [d_month_seq] #2 - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] @@ -58,4 +58,4 @@ WholeStageCodegen (1) CometHashAggregate [i_category,sum,count,i_current_price] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt index 27e71a746b..1990c685ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt @@ -46,10 +46,10 @@ : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,51 +62,51 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) : : : : : : : : : : : : : : +- CometBroadcastExchange (39) : : : : : : : : : : : : : : +- CometProject (38) : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) : : : : : : : : : : : : : +- CometBroadcastExchange (44) : : : : : : : : : : : : : +- CometFilter (43) - : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) : : : : : : : : : : : : +- CometBroadcastExchange (49) : : : : : : : : : : : : +- CometFilter (48) - : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) : : : : : : : : : : : +- ReusedExchange (52) : : : : : : : : : : +- CometBroadcastExchange (58) : : : : : : : : : : +- CometProject (57) : : : : : : : : : : +- CometFilter (56) - : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) : : : : : : : : : +- ReusedExchange (61) : : : : : : : : +- CometBroadcastExchange (66) : : : : : : : : +- CometFilter (65) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.promotion (64) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) : : : : : : : +- CometBroadcastExchange (71) : : : : : : : +- CometFilter (70) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (69) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) : : : : : : +- ReusedExchange (74) : : : : : +- CometBroadcastExchange (80) : : : : : +- CometProject (79) : : : : : +- CometFilter (78) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_address (77) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) : : : : +- ReusedExchange (83) : : : +- CometBroadcastExchange (88) : : : +- CometFilter (87) - : : : +- CometScan [native_comet] parquet spark_catalog.default.income_band (86) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) : : +- ReusedExchange (91) : +- CometBroadcastExchange (97) : +- CometProject (96) : +- CometFilter (95) - : +- CometScan [native_comet] parquet spark_catalog.default.item (94) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) +- CometSort (171) +- CometExchange (170) +- CometHashAggregate (169) @@ -149,10 +149,10 @@ : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (110) : : : : : : : : : : : : : : : : :- CometBroadcastExchange (106) : : : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (104) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (107) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) : : : : : : : : : : : : : : : +- CometSort (118) : : : : : : : : : : : : : : : +- CometProject (117) : : : : : : : : : : : : : : : +- CometFilter (116) @@ -160,7 +160,7 @@ : : : : : : : : : : : : : : : +- ReusedExchange (114) : : : : : : : : : : : : : : +- CometBroadcastExchange (123) : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (121) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) : : : : : : : : : : : : : +- ReusedExchange (126) : : : : : : : : : : : : +- ReusedExchange (129) : : : : : : : : : : : +- ReusedExchange (132) @@ -177,7 +177,7 @@ +- ReusedExchange (165) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -193,7 +193,7 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometScan [native_comet] parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -225,7 +225,7 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -248,7 +248,7 @@ Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] -(16) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -315,7 +315,7 @@ Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometScan [native_comet] parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -339,7 +339,7 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -(36) CometScan [native_comet] parquet spark_catalog.default.store +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -367,7 +367,7 @@ Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -(42) CometScan [native_comet] parquet spark_catalog.default.customer +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -391,7 +391,7 @@ Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -(47) CometScan [native_comet] parquet spark_catalog.default.date_dim +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -427,7 +427,7 @@ Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(55) CometScan [native_comet] parquet spark_catalog.default.customer_demographics +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#48, cd_marital_status#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -467,7 +467,7 @@ Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_stat Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(64) CometScan [native_comet] parquet spark_catalog.default.promotion +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#53] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -491,7 +491,7 @@ Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -(69) CometScan [native_comet] parquet spark_catalog.default.household_demographics +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -527,7 +527,7 @@ Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -(77) CometScan [native_comet] parquet spark_catalog.default.customer_address +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -567,7 +567,7 @@ Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(86) CometScan [native_comet] parquet spark_catalog.default.income_band +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#70] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] @@ -603,7 +603,7 @@ Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -(94) CometScan [native_comet] parquet spark_catalog.default.item +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -649,7 +649,7 @@ Arguments: hashpartitioning(item_sk#82, store_name#83, store_zip#84, 5), ENSURE_ Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] Arguments: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97], [item_sk#82 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, store_zip#84 ASC NULLS FIRST] -(104) CometScan [native_comet] parquet spark_catalog.default.store_sales +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] Batched: true Location: InMemoryFileIndex [] @@ -665,7 +665,7 @@ Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(107) CometScan [native_comet] parquet spark_catalog.default.store_returns +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -726,7 +726,7 @@ Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -(121) CometScan [native_comet] parquet spark_catalog.default.date_dim +(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -962,10 +962,10 @@ Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dy BroadcastExchange (180) +- * CometColumnarToRow (179) +- CometFilter (178) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (177) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) -(177) CometScan [native_comet] parquet spark_catalog.default.date_dim +(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -987,10 +987,10 @@ Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN BroadcastExchange (184) +- * CometColumnarToRow (183) +- CometFilter (182) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (181) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) -(181) CometScan [native_comet] parquet spark_catalog.default.date_dim +(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#122, d_year#123] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt index 0c9153cd68..1dc9f0e3a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt @@ -47,17 +47,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -70,51 +70,51 @@ WholeStageCodegen (1) CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #12 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #12 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #13 CometBroadcastExchange [p_promo_sk] #14 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan [native_comet] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 CometBroadcastExchange [ib_income_band_sk] #17 CometFilter [ib_income_band_sk] - CometScan [native_comet] parquet spark_catalog.default.income_band [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] ReusedExchange [ib_income_band_sk] #17 CometBroadcastExchange [i_item_sk,i_product_name] #18 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] @@ -157,17 +157,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #22 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] @@ -175,7 +175,7 @@ WholeStageCodegen (1) ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 CometBroadcastExchange [d_date_sk,d_year] #23 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #10 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 ReusedExchange [d_date_sk,d_year] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt index 7a2e6dca66..ac04f86f96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt @@ -16,19 +16,19 @@ TakeOrderedAndProject (70) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan [native_comet] parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan [native_comet] parquet spark_catalog.default.item (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) :- CometHashAggregate (28) : +- CometExchange (27) : +- CometHashAggregate (26) @@ -71,7 +71,7 @@ TakeOrderedAndProject (70) +- ReusedExchange (59) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -109,7 +109,7 @@ Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan [native_comet] parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_county#8, s_state#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -123,7 +123,7 @@ Condition : isnotnull(s_store_sk#7) (12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#7, s_county#8, s_state#9] -(13) CometScan [native_comet] parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -135,7 +135,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] Condition : isnotnull(ss_store_sk#10) -(15) CometScan [native_comet] parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#14, s_state#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -373,10 +373,10 @@ BroadcastExchange (70) +- * CometColumnarToRow (69) +- CometProject (68) +- CometFilter (67) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (66) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) -(66) CometScan [native_comet] parquet spark_catalog.default.date_dim +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt index a2389a6a10..72d0bd53d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -36,11 +36,11 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #6 WholeStageCodegen (3) @@ -49,7 +49,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometColumnarToRow InputAdapter CometFilter [s_store_sk,s_county,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter BroadcastExchange #7 WholeStageCodegen (2) @@ -69,12 +69,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometProject [ss_net_profit,ss_sold_date_sk,s_state] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] ReusedSubquery [d_date_sk] #1 CometBroadcastExchange [s_store_sk,s_state] #9 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] - CometScan [native_comet] parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #5 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt index 16ce320133..ee3242f2b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt @@ -27,45 +27,45 @@ : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.inventory (3) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.warehouse (8) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : : : : : : : +- CometBroadcastExchange (15) : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (13) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) : : : : : : +- CometBroadcastExchange (21) : : : : : : +- CometProject (20) : : : : : : +- CometFilter (19) - : : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer_demographics (18) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) : : : : : +- CometBroadcastExchange (27) : : : : : +- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.household_demographics (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) : : : : +- CometBroadcastExchange (33) : : : : +- CometProject (32) : : : : +- CometFilter (31) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : : : +- CometBroadcastExchange (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) : : +- CometBroadcastExchange (43) : : +- CometFilter (42) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) : +- CometBroadcastExchange (48) : +- CometFilter (47) - : +- CometScan [native_comet] parquet spark_catalog.default.promotion (46) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) +- CometSort (57) +- CometExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (53) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -77,7 +77,7 @@ ReadSchema: struct date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(46) CometScan [native_comet] parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#30] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan [native_comet] parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -374,10 +374,10 @@ BroadcastExchange (69) +- * CometColumnarToRow (68) +- CometProject (67) +- CometFilter (66) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) -(65) CometScan [native_comet] parquet spark_catalog.default.date_dim +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt index a89be99fc8..c770348a8e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt @@ -28,7 +28,7 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) @@ -36,39 +36,39 @@ WholeStageCodegen (1) InputAdapter CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan [native_comet] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan [native_comet] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #7 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan [native_comet] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan [native_comet] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #10 CometFilter [d_date_sk,d_week_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #11 CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #12 CometFilter [p_promo_sk] - CometScan [native_comet] parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #13 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan [native_comet] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt index 000db1f74c..b0ae8bb695 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt @@ -16,13 +16,13 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometBroadcastExchange (34) : : +- CometHashAggregate (33) : : +- CometExchange (32) @@ -33,13 +33,13 @@ : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (20) : : : : +- CometFilter (19) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) : : +- CometBroadcastExchange (28) : : +- CometFilter (27) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) : +- CometBroadcastExchange (51) : +- CometFilter (50) : +- CometHashAggregate (49) @@ -51,10 +51,10 @@ : : +- CometBroadcastHashJoin (42) : : :- CometProject (38) : : : +- CometFilter (37) - : : : +- CometScan [native_comet] parquet spark_catalog.default.customer (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) : : +- CometBroadcastExchange (41) : : +- CometFilter (40) - : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) : +- ReusedExchange (44) +- CometBroadcastExchange (68) +- CometHashAggregate (67) @@ -66,14 +66,14 @@ : +- CometBroadcastHashJoin (60) : :- CometProject (56) : : +- CometFilter (55) - : : +- CometScan [native_comet] parquet spark_catalog.default.customer (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) : +- CometBroadcastExchange (59) : +- CometFilter (58) - : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) +- ReusedExchange (62) -(1) CometScan [native_comet] parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -88,7 +88,7 @@ Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apa Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) CometScan [native_comet] parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -113,7 +113,7 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -155,7 +155,7 @@ Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] Input [2]: [customer_id#15, year_total#16] Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) -(18) CometScan [native_comet] parquet spark_catalog.default.customer +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -170,7 +170,7 @@ Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_comet] parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -195,7 +195,7 @@ Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -242,7 +242,7 @@ Left output [2]: [customer_id#15, year_total#16] Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight -(36) CometScan [native_comet] parquet spark_catalog.default.customer +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -257,7 +257,7 @@ Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] -(39) CometScan [native_comet] parquet spark_catalog.default.web_sales +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] @@ -325,7 +325,7 @@ Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -(54) CometScan [native_comet] parquet spark_catalog.default.customer +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -340,7 +340,7 @@ Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.ap Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] -(57) CometScan [native_comet] parquet spark_catalog.default.web_sales +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -417,10 +417,10 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dy BroadcastExchange (76) +- * CometColumnarToRow (75) +- CometFilter (74) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(73) CometScan [native_comet] parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -442,10 +442,10 @@ Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN d BroadcastExchange (80) +- * CometColumnarToRow (79) +- CometFilter (78) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (77) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) -(77) CometScan [native_comet] parquet spark_catalog.default.date_dim +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt index d4c76e024d..400468d3e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt @@ -17,20 +17,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #3 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 @@ -41,20 +41,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #2 BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] @@ -66,10 +66,10 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 @@ -82,9 +82,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan [native_comet] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index 1547bac90e..6925eb301e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -22,19 +22,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -44,14 +44,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -61,14 +61,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan [native_comet] parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -88,11 +88,11 @@ : : :- CometProject (71) : : : +- CometBroadcastHashJoin (70) : : : :- CometFilter (68) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.catalog_sales (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan [native_comet] parquet spark_catalog.default.date_dim (72) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -104,7 +104,7 @@ : : :- CometProject (87) : : : +- CometBroadcastHashJoin (86) : : : :- CometFilter (84) - : : : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (83) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) : : : +- ReusedExchange (85) : : +- ReusedExchange (88) : +- CometSort (94) @@ -118,14 +118,14 @@ : :- CometProject (101) : : +- CometBroadcastHashJoin (100) : : :- CometFilter (98) - : : : +- CometScan [native_comet] parquet spark_catalog.default.web_sales (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) : : +- ReusedExchange (99) : +- ReusedExchange (102) +- CometSort (108) +- ReusedExchange (107) -(1) CometScan [native_comet] parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -137,7 +137,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -78,7 +78,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan [native_comet] parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -213,10 +213,10 @@ BroadcastExchange (42) +- * CometColumnarToRow (41) +- CometProject (40) +- CometFilter (39) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) -(38) CometScan [native_comet] parquet spark_catalog.default.date_dim +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt index d976ac8da8..9c46912086 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt @@ -20,7 +20,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -28,15 +28,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #6 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] CometExchange [i_category] #7 CometHashAggregate [i_category,sum,isEmpty,total_sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt index 91c8e15e33..96a0ec392e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt @@ -15,18 +15,18 @@ :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan [native_comet] parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan [native_comet] parquet spark_catalog.default.item (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) -(1) CometScan [native_comet] parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -38,7 +38,7 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan [native_comet] parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -66,7 +66,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -(9) CometScan [native_comet] parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -145,10 +145,10 @@ BroadcastExchange (30) +- * CometColumnarToRow (29) +- CometProject (28) +- CometFilter (27) - +- CometScan [native_comet] parquet spark_catalog.default.date_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) -(26) CometScan [native_comet] parquet spark_catalog.default.date_dim +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt index 25073f98c7..196e3744a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt @@ -20,7 +20,7 @@ WholeStageCodegen (3) CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan [native_comet] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 BroadcastExchange #4 WholeStageCodegen (1) @@ -28,12 +28,12 @@ WholeStageCodegen (3) InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan [native_comet] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan [native_comet] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index aa80251490..ad2e07c173 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -83,7 +83,7 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa name } val nativeImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() - if (!nativeImpl.equals(CometConf.SCAN_NATIVE_COMET)) { + if (nativeImpl != CometConf.SCAN_AUTO) { goldenFileName = s"$goldenFileName.$nativeImpl" } new File(goldenFilePath, goldenFileName) @@ -259,20 +259,15 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa s"$tpcdsGroup/$query.sql", classLoader = Thread.currentThread().getContextClassLoader) - // Comet does not yet support DPP yet with full native scan enabled - // https://github.com/apache/datafusion-comet/issues/121 - val dppEnabled = CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_NATIVE_COMET - - // Disable char/varchar read-side handling for better performance. withSQLConf( CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true", CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_DPP_FALLBACK_ENABLED.key -> "false", - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> dppEnabled.toString, CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key -> "true", + CometConf.COMET_ANSI_MODE_ENABLED.key -> "true", // needed for Spark 4.0.0 CometConf.COMET_CAST_ALLOW_INCOMPATIBLE.key -> "true", // needed for v1.4/q9, v1.4/q44, v2.7.0/q6, v2.7.0/q64 SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10MB") { val qe = sql(queryString).queryExecution @@ -300,7 +295,7 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa conf.set(CometConf.COMET_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_ENABLED.key, "true") conf.set(CometConf.COMET_NATIVE_SCAN_ENABLED.key, "true") - conf.set(CometConf.COMET_NATIVE_SCAN_IMPL.key, CometConf.SCAN_NATIVE_COMET) + conf.set(CometConf.COMET_NATIVE_SCAN_IMPL.key, CometConf.SCAN_AUTO) conf.set(CometConf.COMET_MEMORY_OVERHEAD.key, "1g") conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") @@ -319,7 +314,7 @@ class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { override val goldenFilePath: String = new File(baseResourcePath, planName).getAbsolutePath - if (CometConf.COMET_NATIVE_SCAN_IMPL.get().equals(CometConf.SCAN_NATIVE_COMET)) { + if (CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_AUTO) { tpcdsQueries.foreach { q => test(s"check simplified (tpcds-v1.4/$q)") { testQuery("tpcds", q) @@ -339,7 +334,7 @@ class CometTPCDSV2_7_PlanStabilitySuite extends CometPlanStabilitySuite { override val goldenFilePath: String = new File(baseResourcePath, planName).getAbsolutePath - if (CometConf.COMET_NATIVE_SCAN_IMPL.get().equals(CometConf.SCAN_NATIVE_COMET)) { + if (CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_AUTO) { tpcdsQueriesV2_7_0.foreach { q => test(s"check simplified (tpcds-v2.7.0/$q)") { testQuery("tpcds-v2.7.0", q)