From b5f505dfec9a298447ae0f8d44ef22de6443fbe1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 12:40:15 -0600 Subject: [PATCH 01/14] Enable more Spark SQL tests for Spark 3.5.6 --- dev/diffs/3.5.6.diff | 100 +------------------------------------------ 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/dev/diffs/3.5.6.diff b/dev/diffs/3.5.6.diff index e3ba0a35c3..7d012e4632 100644 --- a/dev/diffs/3.5.6.diff +++ b/dev/diffs/3.5.6.diff @@ -263,7 +263,7 @@ index 56e9520fdab..917932336df 100644 spark.range(100).write.saveAsTable(s"$dbName.$table2Name") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala -index 7ee18df3756..64f01a68048 100644 +index 7ee18df3756..d09f70e5d99 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -40,11 +40,12 @@ import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation @@ -322,16 +322,6 @@ index 7ee18df3756..64f01a68048 100644 } assert(exchanges.size == 2) } -@@ -3350,7 +3353,8 @@ class DataFrameSuite extends QueryTest - assert(df2.isLocal) - } - -- test("SPARK-35886: PromotePrecision should be subexpr replaced") { -+ test("SPARK-35886: PromotePrecision should be subexpr replaced", -+ IgnoreComet("TODO: fix Comet for this test")) { - withTable("tbl") { - sql( - """ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala index 47a311c71d5..342e71cfdd4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala @@ -2577,94 +2567,6 @@ index 1954cce7fdc..73d1464780e 100644 }.headOption.getOrElse { fail(s"No FileScan in query\n${df.queryExecution}") } -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala -index b597a244710..b2e8be41065 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala -@@ -21,6 +21,7 @@ import java.io.File - - import org.apache.commons.io.FileUtils - -+import org.apache.spark.sql.IgnoreComet - import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update - import org.apache.spark.sql.execution.streaming.{FlatMapGroupsWithStateExec, MemoryStream} - import org.apache.spark.sql.internal.SQLConf -@@ -91,7 +92,7 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest - } - - test("SPARK-38204: flatMapGroupsWithState should require StatefulOpClusteredDistribution " + -- "from children - without initial state") { -+ "from children - without initial state", IgnoreComet("TODO: fix Comet for this test")) { - // function will return -1 on timeout and returns count of the state otherwise - val stateFunc = - (key: (String, String), values: Iterator[(String, String, Long)], -@@ -243,7 +244,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest - } - - test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " + -- "from children if the query starts from checkpoint in 3.2.x - without initial state") { -+ "from children if the query starts from checkpoint in 3.2.x - without initial state", -+ IgnoreComet("TODO: fix Comet for this test")) { - // function will return -1 on timeout and returns count of the state otherwise - val stateFunc = - (key: (String, String), values: Iterator[(String, String, Long)], -@@ -335,7 +337,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest - } - - test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " + -- "from children if the query starts from checkpoint in prior to 3.2") { -+ "from children if the query starts from checkpoint in prior to 3.2", -+ IgnoreComet("TODO: fix Comet for this test")) { - // function will return -1 on timeout and returns count of the state otherwise - val stateFunc = - (key: (String, String), values: Iterator[(String, String, Long)], -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala -index a3774bf17e6..6879c71037d 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala -@@ -25,7 +25,7 @@ import org.scalatest.exceptions.TestFailedException - - import org.apache.spark.SparkException - import org.apache.spark.api.java.function.FlatMapGroupsWithStateFunction --import org.apache.spark.sql.{DataFrame, Encoder} -+import org.apache.spark.sql.{DataFrame, Encoder, IgnoreCometSuite} - import org.apache.spark.sql.catalyst.InternalRow - import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow} - import org.apache.spark.sql.catalyst.plans.logical.FlatMapGroupsWithState -@@ -46,8 +46,9 @@ case class RunningCount(count: Long) - - case class Result(key: Long, count: Int) - -+// TODO: fix Comet to enable this suite - @SlowSQLTest --class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest { -+class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest with IgnoreCometSuite { - - import testImplicits._ - -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala -index 2a2a83d35e1..e3b7b290b3e 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala -@@ -18,7 +18,7 @@ - package org.apache.spark.sql.streaming - - import org.apache.spark.SparkException --import org.apache.spark.sql.{AnalysisException, Dataset, KeyValueGroupedDataset} -+import org.apache.spark.sql.{AnalysisException, Dataset, IgnoreComet, KeyValueGroupedDataset} - import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update - import org.apache.spark.sql.execution.streaming.MemoryStream - import org.apache.spark.sql.execution.streaming.state.FlatMapGroupsWithStateExecHelper -@@ -253,7 +253,8 @@ class FlatMapGroupsWithStateWithInitialStateSuite extends StateStoreMetricsTest - assert(e.message.contains(expectedError)) - } - -- test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState") { -+ test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState", -+ IgnoreComet("TODO: fix Comet for this test")) { - val initialStateDS = Seq(("keyInStateAndData", new RunningCount(1))).toDS() - val initialState: KeyValueGroupedDataset[String, RunningCount] = - initialStateDS.groupByKey(_._1).mapValues(_._2) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala index c97979a57a5..45a998db0e0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala From 7c263a4fc90ea14a4fb6d790c3db707e746dfb8b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 12:58:38 -0600 Subject: [PATCH 02/14] Enable SparkSessionExtensionSuite tests --- dev/diffs/3.5.6.diff | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/dev/diffs/3.5.6.diff b/dev/diffs/3.5.6.diff index 7d012e4632..0da58d8b7d 100644 --- a/dev/diffs/3.5.6.diff +++ b/dev/diffs/3.5.6.diff @@ -879,37 +879,6 @@ index 793a0da6a86..6ccb9d62582 100644 AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") { sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect() } -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -index 8b4ac474f87..3f79f20822f 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -@@ -223,6 +223,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt - withSession(extensions) { session => - session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true) - session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ session.conf.set("spark.comet.enabled", false) - assert(session.sessionState.columnarRules.contains( - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) - import session.sqlContext.implicits._ -@@ -281,6 +283,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt - } - withSession(extensions) { session => - session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE) -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ session.conf.set("spark.comet.enabled", false) - assert(session.sessionState.columnarRules.contains( - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) - import session.sqlContext.implicits._ -@@ -319,6 +323,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt - val session = SparkSession.builder() - .master("local[1]") - .config(COLUMN_BATCH_SIZE.key, 2) -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ .config("spark.comet.enabled", false) - .withExtensions { extensions => - extensions.injectColumnar(session => - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index fa1a64460fc..1d2e215d6a3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala From 3f1abeb4a57290bdeefe9da04712c60c8f049d95 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 13:04:48 -0600 Subject: [PATCH 03/14] update 3.4.3 --- dev/diffs/3.4.3.diff | 139 +--------- native/Cargo.lock | 631 ++++++++++++------------------------------- 2 files changed, 173 insertions(+), 597 deletions(-) diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index dda075f02f..62dfd952ac 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -284,7 +284,7 @@ index 56e9520fdab..917932336df 100644 spark.range(100).write.saveAsTable(s"$dbName.$table2Name") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala -index a9f69ab28a1..5d9d4f2cb83 100644 +index a9f69ab28a1..760ea0e9565 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -39,11 +39,12 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, Attri @@ -343,16 +343,6 @@ index a9f69ab28a1..5d9d4f2cb83 100644 } assert(exchanges.size == 2) } -@@ -3325,7 +3328,8 @@ class DataFrameSuite extends QueryTest - assert(df2.isLocal) - } - -- test("SPARK-35886: PromotePrecision should be subexpr replaced") { -+ test("SPARK-35886: PromotePrecision should be subexpr replaced", -+ IgnoreComet("TODO: fix Comet for this test")) { - withTable("tbl") { - sql( - """ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala index 433b4741979..07148eee480 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala @@ -904,37 +894,6 @@ index 525d97e4998..8a3e7457618 100644 AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") { sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect() } -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -index 48ad10992c5..51d1ee65422 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -@@ -221,6 +221,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper { - withSession(extensions) { session => - session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true) - session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ session.conf.set("spark.comet.enabled", false) - assert(session.sessionState.columnarRules.contains( - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) - import session.sqlContext.implicits._ -@@ -279,6 +281,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper { - } - withSession(extensions) { session => - session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE) -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ session.conf.set("spark.comet.enabled", false) - assert(session.sessionState.columnarRules.contains( - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) - import session.sqlContext.implicits._ -@@ -317,6 +321,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper { - val session = SparkSession.builder() - .master("local[1]") - .config(COLUMN_BATCH_SIZE.key, 2) -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ .config("spark.comet.enabled", false) - .withExtensions { extensions => - extensions.injectColumnar(session => - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index 18123a4d6ec..fbe4c766eee 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala @@ -2520,94 +2479,18 @@ index 75f440caefc..36b1146bc3a 100644 }.headOption.getOrElse { fail(s"No FileScan in query\n${df.queryExecution}") } -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala -index b597a244710..b2e8be41065 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala -@@ -21,6 +21,7 @@ import java.io.File - - import org.apache.commons.io.FileUtils - -+import org.apache.spark.sql.IgnoreComet - import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update - import org.apache.spark.sql.execution.streaming.{FlatMapGroupsWithStateExec, MemoryStream} - import org.apache.spark.sql.internal.SQLConf -@@ -91,7 +92,7 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest - } - - test("SPARK-38204: flatMapGroupsWithState should require StatefulOpClusteredDistribution " + -- "from children - without initial state") { -+ "from children - without initial state", IgnoreComet("TODO: fix Comet for this test")) { - // function will return -1 on timeout and returns count of the state otherwise - val stateFunc = - (key: (String, String), values: Iterator[(String, String, Long)], -@@ -243,7 +244,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest - } - - test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " + -- "from children if the query starts from checkpoint in 3.2.x - without initial state") { -+ "from children if the query starts from checkpoint in 3.2.x - without initial state", -+ IgnoreComet("TODO: fix Comet for this test")) { - // function will return -1 on timeout and returns count of the state otherwise - val stateFunc = - (key: (String, String), values: Iterator[(String, String, Long)], -@@ -335,7 +337,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest - } - - test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " + -- "from children if the query starts from checkpoint in prior to 3.2") { -+ "from children if the query starts from checkpoint in prior to 3.2", -+ IgnoreComet("TODO: fix Comet for this test")) { - // function will return -1 on timeout and returns count of the state otherwise - val stateFunc = - (key: (String, String), values: Iterator[(String, String, Long)], diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala -index 6aa7d0945c7..38523536154 100644 +index 6aa7d0945c7..ad26ad833e2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala -@@ -25,7 +25,7 @@ import org.scalatest.exceptions.TestFailedException - - import org.apache.spark.SparkException - import org.apache.spark.api.java.function.FlatMapGroupsWithStateFunction --import org.apache.spark.sql.{DataFrame, Encoder} -+import org.apache.spark.sql.{DataFrame, Encoder, IgnoreCometSuite} - import org.apache.spark.sql.catalyst.InternalRow - import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow} - import org.apache.spark.sql.catalyst.plans.logical.FlatMapGroupsWithState -@@ -46,8 +46,9 @@ case class RunningCount(count: Long) +@@ -46,6 +46,7 @@ case class RunningCount(count: Long) case class Result(key: Long, count: Int) +// TODO: fix Comet to enable this suite @SlowSQLTest --class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest { -+class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest with IgnoreCometSuite { - - import testImplicits._ - -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala -index 2a2a83d35e1..e3b7b290b3e 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala -@@ -18,7 +18,7 @@ - package org.apache.spark.sql.streaming - - import org.apache.spark.SparkException --import org.apache.spark.sql.{AnalysisException, Dataset, KeyValueGroupedDataset} -+import org.apache.spark.sql.{AnalysisException, Dataset, IgnoreComet, KeyValueGroupedDataset} - import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update - import org.apache.spark.sql.execution.streaming.MemoryStream - import org.apache.spark.sql.execution.streaming.state.FlatMapGroupsWithStateExecHelper -@@ -253,7 +253,8 @@ class FlatMapGroupsWithStateWithInitialStateSuite extends StateStoreMetricsTest - assert(e.message.contains(expectedError)) - } + class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest { -- test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState") { -+ test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState", -+ IgnoreComet("TODO: fix Comet for this test")) { - val initialStateDS = Seq(("keyInStateAndData", new RunningCount(1))).toDS() - val initialState: KeyValueGroupedDataset[String, RunningCount] = - initialStateDS.groupByKey(_._1).mapValues(_._2) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala index ef5b8a769fe..84fe1bfabc9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala @@ -2912,20 +2795,6 @@ index 52abd248f3a..7a199931a08 100644 case h: HiveTableScanExec => h.partitionPruningPred.collect { case d: DynamicPruningExpression => d.child } -diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala -index 1966e1e64fd..cde97a0aafe 100644 ---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala -+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala -@@ -656,7 +656,8 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te - Row(3, 4, 4, 3, null) :: Nil) - } - -- test("single distinct multiple columns set") { -+ test("single distinct multiple columns set", -+ IgnoreComet("TODO: fix Comet for this test")) { - checkAnswer( - spark.sql( - """ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala index 07361cfdce9..b4d53dbe900 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala diff --git a/native/Cargo.lock b/native/Cargo.lock index 6fea0b84ee..71662f97fa 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1265,7 +1265,7 @@ dependencies = [ [[package]] name = "datafusion" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "arrow-ipc", @@ -1273,29 +1273,29 @@ dependencies = [ "async-trait", "bytes", "chrono", - "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-optimizer", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", "futures", "itertools 0.14.0", "log", @@ -1311,31 +1311,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "datafusion-catalog" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", -] - [[package]] name = "datafusion-catalog" version = "48.0.0" @@ -1344,15 +1319,15 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", "futures", "itertools 0.14.0", "log", @@ -1364,19 +1339,19 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "async-trait", - "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "log", "object_store", @@ -1472,28 +1447,6 @@ dependencies = [ "twox-hash 2.1.0", ] -[[package]] -name = "datafusion-common" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "ahash", - "arrow", - "arrow-ipc", - "base64", - "half", - "hashbrown 0.14.5", - "indexmap", - "libc", - "log", - "object_store", - "parquet", - "paste", - "sqlparser", - "tokio", - "web-time", -] - [[package]] name = "datafusion-common" version = "48.0.0" @@ -1509,22 +1462,13 @@ dependencies = [ "libc", "log", "object_store", + "parquet", "paste", "sqlparser", "tokio", "web-time", ] -[[package]] -name = "datafusion-common-runtime" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "futures", - "log", - "tokio", -] - [[package]] name = "datafusion-common-runtime" version = "48.0.0" @@ -1538,20 +1482,20 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "async-trait", "bytes", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "glob", "itertools 0.14.0", @@ -1564,51 +1508,24 @@ dependencies = [ "url", ] -[[package]] -name = "datafusion-datasource" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "futures", - "glob", - "itertools 0.14.0", - "log", - "object_store", - "rand 0.9.1", - "tokio", - "url", -] - [[package]] name = "datafusion-datasource-csv" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "object_store", "regex", @@ -1618,21 +1535,21 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "object_store", "serde_json", @@ -1642,23 +1559,23 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", "datafusion-functions-aggregate", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-optimizer", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan", + "datafusion-session", "futures", "itertools 0.14.0", "log", @@ -1669,34 +1586,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-doc" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" - [[package]] name = "datafusion-doc" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" -[[package]] -name = "datafusion-execution" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "arrow", - "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "futures", - "log", - "object_store", - "parking_lot", - "rand 0.9.1", - "tempfile", - "url", -] - [[package]] name = "datafusion-execution" version = "48.0.0" @@ -1704,8 +1598,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "arrow", "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-expr", "futures", "log", "object_store", @@ -1715,25 +1609,6 @@ dependencies = [ "url", ] -[[package]] -name = "datafusion-expr" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "arrow", - "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "indexmap", - "paste", - "serde_json", - "sqlparser", -] - [[package]] name = "datafusion-expr" version = "48.0.0" @@ -1741,37 +1616,25 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "arrow", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", "indexmap", "paste", "serde_json", "sqlparser", ] -[[package]] -name = "datafusion-expr-common" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "indexmap", - "itertools 0.14.0", - "paste", -] - [[package]] name = "datafusion-expr-common" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", "indexmap", "itertools 0.14.0", "paste", @@ -1780,7 +1643,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "arrow-buffer", @@ -1788,12 +1651,12 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hex", "itertools 0.14.0", "log", @@ -1805,62 +1668,26 @@ dependencies = [ "uuid", ] -[[package]] -name = "datafusion-functions" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" -dependencies = [ - "arrow", - "arrow-buffer", - "base64", - "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "hex", - "itertools 0.14.0", - "log", - "rand 0.9.1", - "regex", - "unicode-segmentation", - "uuid", -] - [[package]] name = "datafusion-functions-aggregate" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "half", "log", "paste", ] -[[package]] -name = "datafusion-functions-aggregate-common" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "ahash", - "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", -] - [[package]] name = "datafusion-functions-aggregate-common" version = "48.0.0" @@ -1868,26 +1695,26 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", ] [[package]] name = "datafusion-functions-nested" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "arrow-ord", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-macros", + "datafusion-physical-expr-common", "itertools 0.14.0", "log", "paste", @@ -1896,14 +1723,14 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "async-trait", - "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", "parking_lot", "paste", ] @@ -1911,46 +1738,27 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "log", "paste", ] -[[package]] -name = "datafusion-functions-window-common" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", -] - [[package]] name = "datafusion-functions-window-common" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", -] - -[[package]] -name = "datafusion-macros" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "quote", - "syn 2.0.101", + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] @@ -1958,7 +1766,7 @@ name = "datafusion-macros" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr", "quote", "syn 2.0.101", ] @@ -1966,13 +1774,13 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", "indexmap", "itertools 0.14.0", "log", @@ -1980,27 +1788,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "datafusion-physical-expr" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "ahash", - "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "half", - "hashbrown 0.14.5", - "indexmap", - "itertools 0.14.0", - "log", - "paste", - "petgraph 0.8.1", -] - [[package]] name = "datafusion-physical-expr" version = "48.0.0" @@ -2008,11 +1795,11 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", "indexmap", @@ -2022,19 +1809,6 @@ dependencies = [ "petgraph 0.8.1", ] -[[package]] -name = "datafusion-physical-expr-common" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "ahash", - "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "hashbrown 0.14.5", - "itertools 0.14.0", -] - [[package]] name = "datafusion-physical-expr-common" version = "48.0.0" @@ -2042,8 +1816,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-expr-common", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -2051,47 +1825,18 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "itertools 0.14.0", - "log", -] - -[[package]] -name = "datafusion-physical-plan" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ - "ahash", "arrow", - "arrow-ord", - "arrow-schema", - "async-trait", - "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "futures", - "half", - "hashbrown 0.14.5", - "indexmap", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "itertools 0.14.0", "log", - "parking_lot", - "pin-project-lite", - "tokio", ] [[package]] @@ -2105,13 +1850,13 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.14.5", @@ -2123,29 +1868,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-session" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", -] - [[package]] name = "datafusion-session" version = "48.0.0" @@ -2154,13 +1876,13 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-sql", "futures", "itertools 0.14.0", "log", @@ -2175,28 +1897,13 @@ version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", - "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "log", -] - -[[package]] -name = "datafusion-sql" -version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" -dependencies = [ - "arrow", - "bigdecimal", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", - "indexmap", + "datafusion-catalog", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-macros", "log", - "regex", - "sqlparser", ] [[package]] @@ -2206,8 +1913,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "arrow", "bigdecimal", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common", + "datafusion-expr", "indexmap", "log", "regex", From b298cabd69ae7f5fddcce2378604a0fd1aa524d9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 13:08:36 -0600 Subject: [PATCH 04/14] update 4.0.0-preview1 --- dev/diffs/4.0.0-preview1.diff | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff index a59c85c212..1a21bc6c3e 100644 --- a/dev/diffs/4.0.0-preview1.diff +++ b/dev/diffs/4.0.0-preview1.diff @@ -995,37 +995,6 @@ index 56c364e2084..fc3abd7cdc4 100644 withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", SQLConf.ANSI_ENABLED.key -> "true") { withTable("t") { -diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -index 4d38e360f43..3c272af0b62 100644 ---- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -@@ -223,6 +223,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt - withSession(extensions) { session => - session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true) - session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ session.conf.set("spark.comet.enabled", false) - assert(session.sessionState.columnarRules.contains( - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) - import session.sqlContext.implicits._ -@@ -281,6 +283,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt - } - withSession(extensions) { session => - session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE) -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ session.conf.set("spark.comet.enabled", false) - assert(session.sessionState.columnarRules.contains( - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) - import session.sqlContext.implicits._ -@@ -319,6 +323,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt - val session = SparkSession.builder() - .master("local[1]") - .config(COLUMN_BATCH_SIZE.key, 2) -+ // https://github.com/apache/datafusion-comet/issues/1197 -+ .config("spark.comet.enabled", false) - .withExtensions { extensions => - extensions.injectColumnar(session => - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index 3fc0b572d80..0d87150d446 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala From 3f43da20d67773f74ee42795281d1243d185749b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 13:11:20 -0600 Subject: [PATCH 05/14] revert --- native/Cargo.lock | 631 +++++++++++++++++++++++++++++++++------------- 1 file changed, 462 insertions(+), 169 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 71662f97fa..6fea0b84ee 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1265,7 +1265,7 @@ dependencies = [ [[package]] name = "datafusion" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "arrow-ipc", @@ -1273,29 +1273,29 @@ dependencies = [ "async-trait", "bytes", "chrono", - "datafusion-catalog", + "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "datafusion-functions-aggregate", "datafusion-functions-nested", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "futures", "itertools 0.14.0", "log", @@ -1311,6 +1311,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-catalog" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + [[package]] name = "datafusion-catalog" version = "48.0.0" @@ -1319,15 +1344,15 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "futures", "itertools 0.14.0", "log", @@ -1339,19 +1364,19 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "futures", "log", "object_store", @@ -1450,7 +1475,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", @@ -1469,6 +1494,37 @@ dependencies = [ "web-time", ] +[[package]] +name = "datafusion-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "base64", + "half", + "hashbrown 0.14.5", + "indexmap", + "libc", + "log", + "object_store", + "paste", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "futures", + "log", + "tokio", +] + [[package]] name = "datafusion-common-runtime" version = "48.0.0" @@ -1482,20 +1538,20 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", "bytes", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "futures", "glob", "itertools 0.14.0", @@ -1509,23 +1565,50 @@ dependencies = [ ] [[package]] -name = "datafusion-datasource-csv" +name = "datafusion-datasource" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "chrono", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "rand 0.9.1", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "futures", "object_store", "regex", @@ -1535,21 +1618,21 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "futures", "object_store", "serde_json", @@ -1559,23 +1642,23 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", + "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "futures", "itertools 0.14.0", "log", @@ -1586,11 +1669,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-doc" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" + [[package]] name = "datafusion-doc" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +[[package]] +name = "datafusion-execution" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "arrow", + "dashmap", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.1", + "tempfile", + "url", +] + [[package]] name = "datafusion-execution" version = "48.0.0" @@ -1598,8 +1704,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "arrow", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "futures", "log", "object_store", @@ -1609,6 +1715,25 @@ dependencies = [ "url", ] +[[package]] +name = "datafusion-expr" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "indexmap", + "paste", + "serde_json", + "sqlparser", +] + [[package]] name = "datafusion-expr" version = "48.0.0" @@ -1616,25 +1741,37 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "indexmap", "paste", "serde_json", "sqlparser", ] +[[package]] +name = "datafusion-expr-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "indexmap", + "itertools 0.14.0", + "paste", +] + [[package]] name = "datafusion-expr-common" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "indexmap", "itertools 0.14.0", "paste", @@ -1643,7 +1780,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "arrow-buffer", @@ -1651,12 +1788,12 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "hex", "itertools 0.14.0", "log", @@ -1669,25 +1806,61 @@ dependencies = [ ] [[package]] -name = "datafusion-functions-aggregate" +name = "datafusion-functions" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "hex", + "itertools 0.14.0", + "log", + "rand 0.9.1", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "half", "log", "paste", ] +[[package]] +name = "datafusion-functions-aggregate-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", +] + [[package]] name = "datafusion-functions-aggregate-common" version = "48.0.0" @@ -1695,26 +1868,26 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", ] [[package]] name = "datafusion-functions-nested" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "datafusion-functions-aggregate", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "itertools 0.14.0", "log", "paste", @@ -1723,14 +1896,14 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "parking_lot", "paste", ] @@ -1738,27 +1911,46 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "log", "paste", ] +[[package]] +name = "datafusion-functions-window-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", +] + [[package]] name = "datafusion-functions-window-common" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", +] + +[[package]] +name = "datafusion-macros" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "quote", + "syn 2.0.101", ] [[package]] @@ -1766,7 +1958,7 @@ name = "datafusion-macros" version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ - "datafusion-expr", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "quote", "syn 2.0.101", ] @@ -1774,13 +1966,13 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "indexmap", "itertools 0.14.0", "log", @@ -1788,6 +1980,27 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "datafusion-physical-expr" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.14.0", + "log", + "paste", + "petgraph 0.8.1", +] + [[package]] name = "datafusion-physical-expr" version = "48.0.0" @@ -1795,11 +2008,11 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "half", "hashbrown 0.14.5", "indexmap", @@ -1809,6 +2022,19 @@ dependencies = [ "petgraph 0.8.1", ] +[[package]] +name = "datafusion-physical-expr-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "hashbrown 0.14.5", + "itertools 0.14.0", +] + [[package]] name = "datafusion-physical-expr-common" version = "48.0.0" @@ -1816,8 +2042,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -1825,20 +2051,49 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", "itertools 0.14.0", "log", ] +[[package]] +name = "datafusion-physical-plan" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "ahash", + "arrow", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.14.0", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "datafusion-physical-plan" version = "48.0.0" @@ -1850,13 +2105,13 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "futures", "half", "hashbrown 0.14.5", @@ -1868,6 +2123,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-session" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + [[package]] name = "datafusion-session" version = "48.0.0" @@ -1876,13 +2154,13 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-sql", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "futures", "itertools 0.14.0", "log", @@ -1897,13 +2175,28 @@ version = "48.0.0" source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98" dependencies = [ "arrow", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-macros", + "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "log", +] + +[[package]] +name = "datafusion-sql" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" +dependencies = [ + "arrow", + "bigdecimal", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)", + "indexmap", "log", + "regex", + "sqlparser", ] [[package]] @@ -1913,8 +2206,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7 dependencies = [ "arrow", "bigdecimal", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)", "indexmap", "log", "regex", From a59c22018cb0e457ef88a3d588d09b51c674f996 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 14:50:06 -0600 Subject: [PATCH 06/14] fix regressions --- dev/diffs/3.4.3.diff | 20 ++++++++++++++++++++ dev/diffs/4.0.0-preview1.diff | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index 62dfd952ac..5a7dd6f15c 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -894,6 +894,26 @@ index 525d97e4998..8a3e7457618 100644 AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") { sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect() } +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +index 48ad10992c5..bd3d55d9711 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +@@ -188,11 +188,13 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper { + } + } + +- test("inject columnar AQE on") { ++ test("inject columnar AQE on", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + testInjectColumnar(true) + } + +- test("inject columnar AQE off") { ++ test("inject columnar AQE off", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + testInjectColumnar(false) + } + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index 18123a4d6ec..fbe4c766eee 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff index 1a21bc6c3e..e6cf67c834 100644 --- a/dev/diffs/4.0.0-preview1.diff +++ b/dev/diffs/4.0.0-preview1.diff @@ -995,6 +995,26 @@ index 56c364e2084..fc3abd7cdc4 100644 withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", SQLConf.ANSI_ENABLED.key -> "true") { withTable("t") { +diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +index 4d38e360f43..4fc255a347f 100644 +--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala ++++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +@@ -190,11 +190,13 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt + } + } + +- test("inject columnar AQE on") { ++ test("inject columnar AQE on", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + testInjectColumnar(true) + } + +- test("inject columnar AQE off") { ++ test("inject columnar AQE off", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + testInjectColumnar(false) + } + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index 3fc0b572d80..0d87150d446 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala From a6947ed195e240ccc4f7434f846c419f47c1f8c7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 14:55:40 -0600 Subject: [PATCH 07/14] fix regressions --- dev/diffs/3.4.3.diff | 22 +++++++++++++++++++++- dev/diffs/4.0.0-preview1.diff | 22 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index 5a7dd6f15c..a1ddf393e2 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -895,7 +895,7 @@ index 525d97e4998..8a3e7457618 100644 sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect() } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -index 48ad10992c5..bd3d55d9711 100644 +index 48ad10992c5..3ec91cccfa8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala @@ -188,11 +188,13 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper { @@ -914,6 +914,26 @@ index 48ad10992c5..bd3d55d9711 100644 testInjectColumnar(false) } +@@ -213,7 +215,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper { + } + } + +- test("SPARK-39991: AQE should retain column statistics from completed query stages") { ++ test("SPARK-39991: AQE should retain column statistics from completed query stages", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + val extensions = create { extensions => + extensions.injectColumnar(_ => + MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) +@@ -313,7 +316,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper { + } + } + +- test("reset column vectors") { ++ test("reset column vectors", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + val session = SparkSession.builder() + .master("local[1]") + .config(COLUMN_BATCH_SIZE.key, 2) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index 18123a4d6ec..fbe4c766eee 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff index e6cf67c834..3fad24ec86 100644 --- a/dev/diffs/4.0.0-preview1.diff +++ b/dev/diffs/4.0.0-preview1.diff @@ -996,7 +996,7 @@ index 56c364e2084..fc3abd7cdc4 100644 SQLConf.ANSI_ENABLED.key -> "true") { withTable("t") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala -index 4d38e360f43..4fc255a347f 100644 +index 4d38e360f43..21e88695976 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala @@ -190,11 +190,13 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt @@ -1015,6 +1015,26 @@ index 4d38e360f43..4fc255a347f 100644 testInjectColumnar(false) } +@@ -215,7 +217,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt + } + } + +- test("SPARK-39991: AQE should retain column statistics from completed query stages") { ++ test("SPARK-39991: AQE should retain column statistics from completed query stages", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + val extensions = create { extensions => + extensions.injectColumnar(_ => + MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) +@@ -315,7 +318,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt + } + } + +- test("reset column vectors") { ++ test("reset column vectors", ++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/1197")) { + val session = SparkSession.builder() + .master("local[1]") + .config(COLUMN_BATCH_SIZE.key, 2) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index 3fc0b572d80..0d87150d446 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala From 5a72fa77c84f98bc52034e23efce6fd49fda33df Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 Jun 2025 14:59:03 -0600 Subject: [PATCH 08/14] Stop Running Spark SQL tests for Spark 3.5.4 and 3.5.5 --- .github/workflows/spark_sql_test.yml | 2 +- dev/diffs/3.5.4.diff | 3223 -------------------------- dev/diffs/3.5.5.diff | 3082 ------------------------ 3 files changed, 1 insertion(+), 6306 deletions(-) delete mode 100644 dev/diffs/3.5.4.diff delete mode 100644 dev/diffs/3.5.5.diff diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml index cbe3effbc7..aa5e6bb887 100644 --- a/.github/workflows/spark_sql_test.yml +++ b/.github/workflows/spark_sql_test.yml @@ -45,7 +45,7 @@ jobs: matrix: os: [ubuntu-24.04] java-version: [11] - spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.4'}, {short: '3.5', full: '3.5.5'}, {short: '3.5', full: '3.5.6'}] + spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}] module: - {name: "catalyst", args1: "catalyst/test", args2: ""} - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest} diff --git a/dev/diffs/3.5.4.diff b/dev/diffs/3.5.4.diff deleted file mode 100644 index 3695871752..0000000000 --- a/dev/diffs/3.5.4.diff +++ /dev/null @@ -1,3223 +0,0 @@ -diff --git a/pom.xml b/pom.xml -index 8dc47f391f9..73feb7e07d2 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -152,6 +152,8 @@ - --> - 2.5.1 - 2.0.8 -+ 3.5 -+ 0.9.0-SNAPSHOT - - 2.5.1 - 2.0.8 -+ 3.5 -+ 0.9.0-SNAPSHOT -