diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml
index 8d5ade8766..a0373e15e9 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -74,14 +74,14 @@ jobs:
             maven_opts: "-Pspark-3.4 -Pscala-2.12"
             scan_impl: "native_comet"
 
-          - name: "Spark 3.5, JDK 11, Scala 2.12"
-            java_version: "17"
-            maven_opts: "-Pspark-3.5 -Pscala-2.12"
+          - name: "Spark 3.5.4, JDK 11, Scala 2.12"
+            java_version: "11"
+            maven_opts: "-Pspark-3.5 -Dspark.version=3.5.4 -Pscala-2.12"
             scan_impl: "native_comet"
 
-          - name: "Spark 3.5, JDK 17, Scala 2.13"
+          - name: "Spark 3.5.5, JDK 17, Scala 2.13"
             java_version: "17"
-            maven_opts: "-Pspark-3.5 -Pscala-2.13"
+            maven_opts: "-Pspark-3.5 -Dspark.version=3.5.5 -Pscala-2.13"
             scan_impl: "native_comet"
 
           - name: "Spark 3.5, JDK 17, Scala 2.12 native_datafusion"
diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml
index cbe3effbc7..aa5e6bb887 100644
--- a/.github/workflows/spark_sql_test.yml
+++ b/.github/workflows/spark_sql_test.yml
@@ -45,7 +45,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.4'}, {short: '3.5', full: '3.5.5'}, {short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff
index dda075f02f..52aa73e93d 100644
--- a/dev/diffs/3.4.3.diff
+++ b/dev/diffs/3.4.3.diff
@@ -284,7 +284,7 @@ index 56e9520fdab..917932336df 100644
            spark.range(100).write.saveAsTable(s"$dbName.$table2Name")
  
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-index a9f69ab28a1..5d9d4f2cb83 100644
+index a9f69ab28a1..760ea0e9565 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
 @@ -39,11 +39,12 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, Attri
@@ -343,16 +343,6 @@ index a9f69ab28a1..5d9d4f2cb83 100644
      }
      assert(exchanges.size == 2)
    }
-@@ -3325,7 +3328,8 @@ class DataFrameSuite extends QueryTest
-     assert(df2.isLocal)
-   }
- 
--  test("SPARK-35886: PromotePrecision should be subexpr replaced") {
-+  test("SPARK-35886: PromotePrecision should be subexpr replaced",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     withTable("tbl") {
-       sql(
-         """
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
 index 433b4741979..07148eee480 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
@@ -1900,7 +1890,7 @@ index 07e2849ce6f..3e73645b638 100644
        ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
      )
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 104b4e416cd..81af723b4d0 100644
+index 104b4e416cd..0bd2e24e387 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
 @@ -1096,7 +1096,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1952,17 +1942,7 @@ index 104b4e416cd..81af723b4d0 100644
        }
      }
    }
-@@ -1744,7 +1757,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
-+  test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+      IgnoreComet("IN predicate is not yet supported in Comet, see issue #36")) {
-     val schema = StructType(Seq(
-       StructField("a", IntegerType, nullable = false)
-     ))
-@@ -1985,7 +1999,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
+@@ -1985,7 +1998,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
      }
    }
  
@@ -1972,7 +1952,7 @@ index 104b4e416cd..81af723b4d0 100644
      // block 1:
      //                      null count  min                                       max
      // page-0                         0  0                                         99
-@@ -2045,7 +2060,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
+@@ -2045,7 +2059,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
      }
    }
  
@@ -1982,7 +1962,7 @@ index 104b4e416cd..81af723b4d0 100644
      withTempPath { dir =>
        val path = dir.getCanonicalPath
        spark.range(100).selectExpr("id * 2 AS id")
-@@ -2277,7 +2293,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
+@@ -2277,7 +2292,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
            assert(pushedParquetFilters.exists(_.getClass === filterClass),
              s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
  
@@ -1995,7 +1975,7 @@ index 104b4e416cd..81af723b4d0 100644
          } else {
            assert(selectedFilters.isEmpty, "There is filter pushed down")
          }
-@@ -2337,7 +2357,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
+@@ -2337,7 +2356,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
            assert(pushedParquetFilters.exists(_.getClass === filterClass),
              s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
  
@@ -2520,94 +2500,18 @@ index 75f440caefc..36b1146bc3a 100644
        }.headOption.getOrElse {
          fail(s"No FileScan in query\n${df.queryExecution}")
        }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-index b597a244710..b2e8be41065 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-@@ -21,6 +21,7 @@ import java.io.File
- 
- import org.apache.commons.io.FileUtils
- 
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.{FlatMapGroupsWithStateExec, MemoryStream}
- import org.apache.spark.sql.internal.SQLConf
-@@ -91,7 +92,7 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require StatefulOpClusteredDistribution " +
--    "from children - without initial state") {
-+    "from children - without initial state", IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -243,7 +244,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in 3.2.x - without initial state") {
-+    "from children if the query starts from checkpoint in 3.2.x - without initial state",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -335,7 +337,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in prior to 3.2") {
-+    "from children if the query starts from checkpoint in prior to 3.2",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-index 6aa7d0945c7..38523536154 100644
+index 6aa7d0945c7..ad26ad833e2 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-@@ -25,7 +25,7 @@ import org.scalatest.exceptions.TestFailedException
- 
- import org.apache.spark.SparkException
- import org.apache.spark.api.java.function.FlatMapGroupsWithStateFunction
--import org.apache.spark.sql.{DataFrame, Encoder}
-+import org.apache.spark.sql.{DataFrame, Encoder, IgnoreCometSuite}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow}
- import org.apache.spark.sql.catalyst.plans.logical.FlatMapGroupsWithState
-@@ -46,8 +46,9 @@ case class RunningCount(count: Long)
+@@ -46,6 +46,7 @@ case class RunningCount(count: Long)
  
  case class Result(key: Long, count: Int)
  
 +// TODO: fix Comet to enable this suite
  @SlowSQLTest
--class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
-+class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest with IgnoreCometSuite {
- 
-   import testImplicits._
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-index 2a2a83d35e1..e3b7b290b3e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.streaming
- 
- import org.apache.spark.SparkException
--import org.apache.spark.sql.{AnalysisException, Dataset, KeyValueGroupedDataset}
-+import org.apache.spark.sql.{AnalysisException, Dataset, IgnoreComet, KeyValueGroupedDataset}
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.MemoryStream
- import org.apache.spark.sql.execution.streaming.state.FlatMapGroupsWithStateExecHelper
-@@ -253,7 +253,8 @@ class FlatMapGroupsWithStateWithInitialStateSuite extends StateStoreMetricsTest
-     assert(e.message.contains(expectedError))
-   }
+ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
  
--  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState") {
-+  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     val initialStateDS = Seq(("keyInStateAndData", new RunningCount(1))).toDS()
-     val initialState: KeyValueGroupedDataset[String, RunningCount] =
-       initialStateDS.groupByKey(_._1).mapValues(_._2)
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
 index ef5b8a769fe..84fe1bfabc9 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
@@ -2912,20 +2816,6 @@ index 52abd248f3a..7a199931a08 100644
        case h: HiveTableScanExec => h.partitionPruningPred.collect {
          case d: DynamicPruningExpression => d.child
        }
-diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
-index 1966e1e64fd..cde97a0aafe 100644
---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
-+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
-@@ -656,7 +656,8 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
-         Row(3, 4, 4, 3, null) :: Nil)
-   }
- 
--  test("single distinct multiple columns set") {
-+  test("single distinct multiple columns set",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     checkAnswer(
-       spark.sql(
-         """
 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
 index 07361cfdce9..b4d53dbe900 100644
 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
diff --git a/dev/diffs/3.5.4.diff b/dev/diffs/3.5.4.diff
deleted file mode 100644
index 3695871752..0000000000
--- a/dev/diffs/3.5.4.diff
+++ /dev/null
@@ -1,3223 +0,0 @@
-diff --git a/pom.xml b/pom.xml
-index 8dc47f391f9..73feb7e07d2 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -152,6 +152,8 @@
-     -->
-     <ivy.version>2.5.1</ivy.version>
-     <oro.version>2.0.8</oro.version>
-+    <spark.version.short>3.5</spark.version.short>
-+    <comet.version>0.9.0-SNAPSHOT</comet.version>
-     <!--
-     If you changes codahale.metrics.version, you also need to change
-     the link to metrics.dropwizard.io in docs/monitoring.md.
-@@ -2836,6 +2838,25 @@
-         <artifactId>okio</artifactId>
-         <version>${okio.version}</version>
-       </dependency>
-+      <dependency>
-+        <groupId>org.apache.datafusion</groupId>
-+        <artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
-+        <version>${comet.version}</version>
-+        <exclusions>
-+          <exclusion>
-+            <groupId>org.apache.spark</groupId>
-+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
-+          </exclusion>
-+          <exclusion>
-+            <groupId>org.apache.spark</groupId>
-+            <artifactId>spark-core_${scala.binary.version}</artifactId>
-+          </exclusion>
-+          <exclusion>
-+            <groupId>org.apache.spark</groupId>
-+            <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
-+          </exclusion>
-+        </exclusions>
-+      </dependency>
-     </dependencies>
-   </dependencyManagement>
- 
-diff --git a/sql/core/pom.xml b/sql/core/pom.xml
-index 9577de81c20..a37f4a1f89f 100644
---- a/sql/core/pom.xml
-+++ b/sql/core/pom.xml
-@@ -77,6 +77,10 @@
-       <groupId>org.apache.spark</groupId>
-       <artifactId>spark-tags_${scala.binary.version}</artifactId>
-     </dependency>
-+    <dependency>
-+      <groupId>org.apache.datafusion</groupId>
-+      <artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
-+    </dependency>
- 
-     <!--
-       This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
-diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
-index 27ae10b3d59..78e69902dfd 100644
---- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
-+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
-@@ -1353,6 +1353,14 @@ object SparkSession extends Logging {
-     }
-   }
- 
-+  private def loadCometExtension(sparkContext: SparkContext): Seq[String] = {
-+    if (sparkContext.getConf.getBoolean("spark.comet.enabled", isCometEnabled)) {
-+      Seq("org.apache.comet.CometSparkSessionExtensions")
-+    } else {
-+      Seq.empty
-+    }
-+  }
-+
-   /**
-    * Initialize extensions specified in [[StaticSQLConf]]. The classes will be applied to the
-    * extensions passed into this function.
-@@ -1362,6 +1370,7 @@ object SparkSession extends Logging {
-       extensions: SparkSessionExtensions): SparkSessionExtensions = {
-     val extensionConfClassNames = sparkContext.getConf.get(StaticSQLConf.SPARK_SESSION_EXTENSIONS)
-       .getOrElse(Seq.empty)
-+    val extensionClassNames = extensionConfClassNames ++ loadCometExtension(sparkContext)
-     extensionConfClassNames.foreach { extensionConfClassName =>
-       try {
-         val extensionConfClass = Utils.classForName(extensionConfClassName)
-@@ -1396,4 +1405,12 @@ object SparkSession extends Logging {
-       }
-     }
-   }
-+
-+  /**
-+   * Whether Comet extension is enabled
-+   */
-+  def isCometEnabled: Boolean = {
-+    val v = System.getenv("ENABLE_COMET")
-+    v == null || v.toBoolean
-+  }
- }
-diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
-index db587dd9868..aac7295a53d 100644
---- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
-+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.annotation.DeveloperApi
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
- import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
- import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
-@@ -67,6 +68,7 @@ private[execution] object SparkPlanInfo {
-     // dump the file scan metadata (e.g file path) to event log
-     val metadata = plan match {
-       case fileScan: FileSourceScanExec => fileScan.metadata
-+      case cometScan: CometScanExec => cometScan.metadata
-       case _ => Map[String, String]()
-     }
-     new SparkPlanInfo(
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
-index 7aef901da4f..f3d6e18926d 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
-@@ -2,3 +2,4 @@
- 
- --SET spark.sql.adaptive.enabled=true
- --SET spark.sql.maxMetadataStringLength = 500
-+--SET spark.comet.enabled = false
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
-index eeb2180f7a5..afd1b5ec289 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
-@@ -1,5 +1,6 @@
- --SET spark.sql.cbo.enabled=true
- --SET spark.sql.maxMetadataStringLength = 500
-+--SET spark.comet.enabled = false
- 
- CREATE TABLE explain_temp1(a INT, b INT) USING PARQUET;
- CREATE TABLE explain_temp2(c INT, d INT) USING PARQUET;
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain.sql b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
-index 698ca009b4f..57d774a3617 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/explain.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
-@@ -1,6 +1,7 @@
- --SET spark.sql.codegen.wholeStage = true
- --SET spark.sql.adaptive.enabled = false
- --SET spark.sql.maxMetadataStringLength = 500
-+--SET spark.comet.enabled = false
- 
- -- Test tables
- CREATE table  explain_temp1 (key int, val int) USING PARQUET;
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql
-index 1152d77da0c..f77493f690b 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql
-@@ -7,6 +7,9 @@
- 
- -- avoid bit-exact output here because operations may not be bit-exact.
- -- SET extra_float_digits = 0;
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- 
- -- Test aggregate operator with codegen on and off.
- --CONFIG_DIM1 spark.sql.codegen.wholeStage=true
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
-index 41fd4de2a09..44cd244d3b0 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
-@@ -5,6 +5,9 @@
- -- AGGREGATES [Part 3]
- -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/aggregates.sql#L352-L605
- 
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- -- Test aggregate operator with codegen on and off.
- --CONFIG_DIM1 spark.sql.codegen.wholeStage=true
- --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql
-index fac23b4a26f..2b73732c33f 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql
-@@ -1,6 +1,10 @@
- --
- -- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
- --
-+
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- --
- -- INT8
- -- Test int8 64-bit integers.
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
-index 0efe0877e9b..423d3b3d76d 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
-@@ -1,6 +1,10 @@
- --
- -- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
- --
-+
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- --
- -- SELECT_HAVING
- -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/select_having.sql
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
-index 9815cb816c9..95b5f9992b0 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
-@@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants
- import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, RDDScanExec, SparkPlan}
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQEPropagateEmptyRelation}
- import org.apache.spark.sql.execution.columnar._
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -519,7 +519,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
-       df.collect()
-     }
-     assert(
--      collect(df.queryExecution.executedPlan) { case e: ShuffleExchangeExec => e }.size == expected)
-+      collect(df.queryExecution.executedPlan) {
-+        case _: ShuffleExchangeLike => 1 }.size == expected)
-   }
- 
-   test("A cached table preserves the partitioning and ordering of its cached SparkPlan") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
-index 5a8681aed97..da9d25e2eb4 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
-@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.plans.logical.Expand
- import org.apache.spark.sql.execution.WholeStageCodegenExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.expressions.Window
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -793,7 +793,7 @@ class DataFrameAggregateSuite extends QueryTest
-       assert(objHashAggPlans.nonEmpty)
- 
-       val exchangePlans = collect(aggPlan) {
--        case shuffle: ShuffleExchangeExec => shuffle
-+        case shuffle: ShuffleExchangeLike => shuffle
-       }
-       assert(exchangePlans.length == 1)
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
-index 56e9520fdab..917932336df 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
-@@ -435,7 +435,9 @@ class DataFrameJoinSuite extends QueryTest
- 
-     withTempDatabase { dbName =>
-       withTable(table1Name, table2Name) {
--        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-+        withSQLConf(
-+            SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-+            "spark.comet.enabled" -> "false") {
-           spark.range(50).write.saveAsTable(s"$dbName.$table1Name")
-           spark.range(100).write.saveAsTable(s"$dbName.$table2Name")
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-index 7ee18df3756..64f01a68048 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-@@ -40,11 +40,12 @@ import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
- import org.apache.spark.sql.catalyst.parser.ParseException
- import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LeafNode, LocalRelation, LogicalPlan, OneRowRelation, Statistics}
- import org.apache.spark.sql.catalyst.util.DateTimeUtils
-+import org.apache.spark.sql.comet.CometBroadcastExchangeExec
- import org.apache.spark.sql.connector.FakeV2Provider
- import org.apache.spark.sql.execution.{FilterExec, LogicalRDD, QueryExecution, SortExec, WholeStageCodegenExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.aggregate.HashAggregateExec
--import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
-+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.expressions.{Aggregator, Window}
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -2006,7 +2007,7 @@ class DataFrameSuite extends QueryTest
-           fail("Should not have back to back Aggregates")
-         }
-         atFirstAgg = true
--      case e: ShuffleExchangeExec => atFirstAgg = false
-+      case e: ShuffleExchangeLike => atFirstAgg = false
-       case _ =>
-     }
-   }
-@@ -2330,7 +2331,7 @@ class DataFrameSuite extends QueryTest
-       checkAnswer(join, df)
-       assert(
-         collect(join.queryExecution.executedPlan) {
--          case e: ShuffleExchangeExec => true }.size === 1)
-+          case _: ShuffleExchangeLike => true }.size === 1)
-       assert(
-         collect(join.queryExecution.executedPlan) { case e: ReusedExchangeExec => true }.size === 1)
-       val broadcasted = broadcast(join)
-@@ -2338,10 +2339,12 @@ class DataFrameSuite extends QueryTest
-       checkAnswer(join2, df)
-       assert(
-         collect(join2.queryExecution.executedPlan) {
--          case e: ShuffleExchangeExec => true }.size == 1)
-+          case _: ShuffleExchangeLike => true }.size == 1)
-       assert(
-         collect(join2.queryExecution.executedPlan) {
--          case e: BroadcastExchangeExec => true }.size === 1)
-+          case e: BroadcastExchangeExec => true
-+          case _: CometBroadcastExchangeExec => true
-+        }.size === 1)
-       assert(
-         collect(join2.queryExecution.executedPlan) { case e: ReusedExchangeExec => true }.size == 4)
-     }
-@@ -2901,7 +2904,7 @@ class DataFrameSuite extends QueryTest
- 
-     // Assert that no extra shuffle introduced by cogroup.
-     val exchanges = collect(df3.queryExecution.executedPlan) {
--      case h: ShuffleExchangeExec => h
-+      case h: ShuffleExchangeLike => h
-     }
-     assert(exchanges.size == 2)
-   }
-@@ -3350,7 +3353,8 @@ class DataFrameSuite extends QueryTest
-     assert(df2.isLocal)
-   }
- 
--  test("SPARK-35886: PromotePrecision should be subexpr replaced") {
-+  test("SPARK-35886: PromotePrecision should be subexpr replaced",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     withTable("tbl") {
-       sql(
-         """
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-index 47a311c71d5..342e71cfdd4 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-@@ -24,8 +24,9 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression
- import org.apache.spark.sql.catalyst.optimizer.TransposeWindow
- import org.apache.spark.sql.catalyst.plans.logical.{Window => LogicalWindow}
- import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
-+import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
--import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, Exchange, ShuffleExchangeExec}
-+import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, Exchange, ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.window.WindowExec
- import org.apache.spark.sql.expressions.{Aggregator, MutableAggregationBuffer, UserDefinedAggregateFunction, Window}
- import org.apache.spark.sql.functions._
-@@ -1187,10 +1188,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest
-     }
- 
-     def isShuffleExecByRequirement(
--        plan: ShuffleExchangeExec,
-+        plan: ShuffleExchangeLike,
-         desiredClusterColumns: Seq[String]): Boolean = plan match {
-       case ShuffleExchangeExec(op: HashPartitioning, _, ENSURE_REQUIREMENTS, _) =>
-         partitionExpressionsColumns(op.expressions) === desiredClusterColumns
-+      case CometShuffleExchangeExec(op: HashPartitioning, _, _, ENSURE_REQUIREMENTS, _, _) =>
-+        partitionExpressionsColumns(op.expressions) === desiredClusterColumns
-       case _ => false
-     }
- 
-@@ -1213,7 +1216,7 @@ class DataFrameWindowFunctionsSuite extends QueryTest
-       val shuffleByRequirement = windowed.queryExecution.executedPlan.exists {
-         case w: WindowExec =>
-           w.child.exists {
--            case s: ShuffleExchangeExec => isShuffleExecByRequirement(s, Seq("key1", "key2"))
-+            case s: ShuffleExchangeLike => isShuffleExecByRequirement(s, Seq("key1", "key2"))
-             case _ => false
-           }
-         case _ => false
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
-index f32b32ffc5a..447d7c6416e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
-@@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
- import org.apache.spark.sql.catalyst.util.sideBySide
- import org.apache.spark.sql.execution.{LogicalRDD, RDDScanExec, SQLExecution}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
--import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec}
-+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.streaming.MemoryStream
- import org.apache.spark.sql.expressions.UserDefinedFunction
- import org.apache.spark.sql.functions._
-@@ -2288,7 +2288,7 @@ class DatasetSuite extends QueryTest
- 
-     // Assert that no extra shuffle introduced by cogroup.
-     val exchanges = collect(df3.queryExecution.executedPlan) {
--      case h: ShuffleExchangeExec => h
-+      case h: ShuffleExchangeLike => h
-     }
-     assert(exchanges.size == 2)
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
-index f33432ddb6f..0fa49fb3f0b 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
-@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
- import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
- import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
- import org.apache.spark.sql.catalyst.plans.ExistenceJoin
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.connector.catalog.{InMemoryTableCatalog, InMemoryTableWithV2FilterCatalog}
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.adaptive._
-@@ -262,6 +263,9 @@ abstract class DynamicPartitionPruningSuiteBase
-       case s: BatchScanExec => s.runtimeFilters.collect {
-         case d: DynamicPruningExpression => d.child
-       }
-+      case s: CometScanExec => s.partitionFilters.collect {
-+        case d: DynamicPruningExpression => d.child
-+      }
-       case _ => Nil
-     }
-   }
-@@ -665,7 +669,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("partition pruning in broadcast hash joins with aliases") {
-+  test("partition pruning in broadcast hash joins with aliases",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     Given("alias with simple join condition, using attribute names only")
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       val df = sql(
-@@ -755,7 +760,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("partition pruning in broadcast hash joins") {
-+  test("partition pruning in broadcast hash joins",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     Given("disable broadcast pruning and disable subquery duplication")
-     withSQLConf(
-       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
-@@ -990,7 +996,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("different broadcast subqueries with identical children") {
-+  test("different broadcast subqueries with identical children",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       withTable("fact", "dim") {
-         spark.range(100).select(
-@@ -1027,7 +1034,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("avoid reordering broadcast join keys to match input hash partitioning") {
-+  test("avoid reordering broadcast join keys to match input hash partitioning",
-+    IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-       withTable("large", "dimTwo", "dimThree") {
-@@ -1187,7 +1195,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("Make sure dynamic pruning works on uncorrelated queries") {
-+  test("Make sure dynamic pruning works on uncorrelated queries",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       val df = sql(
-         """
-@@ -1215,7 +1224,8 @@ abstract class DynamicPartitionPruningSuiteBase
-   }
- 
-   test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
--    "canonicalization and exchange reuse") {
-+    "canonicalization and exchange reuse",
-+    IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-         val df = sql(
-@@ -1238,7 +1248,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("Plan broadcast pruning only when the broadcast can be reused") {
-+  test("Plan broadcast pruning only when the broadcast can be reused",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     Given("dynamic pruning filter on the build side")
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       val df = sql(
-@@ -1279,7 +1290,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") {
-+  test("SPARK-32659: Fix the data issue when pruning DPP on non-atomic type",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     Seq(NO_CODEGEN, CODEGEN_ONLY).foreach { mode =>
-       Seq(true, false).foreach { pruning =>
-         withSQLConf(
-@@ -1311,7 +1323,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-32817: DPP throws error when the broadcast side is empty") {
-+  test("SPARK-32817: DPP throws error when the broadcast side is empty",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(
-       SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
-@@ -1423,7 +1436,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-34637: DPP side broadcast query stage is created firstly") {
-+  test("SPARK-34637: DPP side broadcast query stage is created firstly",
-+    IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       val df = sql(
-         """ WITH v as (
-@@ -1454,7 +1468,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-35568: Fix UnsupportedOperationException when enabling both AQE and DPP") {
-+  test("SPARK-35568: Fix UnsupportedOperationException when enabling both AQE and DPP",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     val df = sql(
-       """
-         |SELECT s.store_id, f.product_id
-@@ -1470,7 +1485,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     checkAnswer(df, Row(3, 2) :: Row(3, 2) :: Row(3, 2) :: Row(3, 2) :: Nil)
-   }
- 
--  test("SPARK-36444: Remove OptimizeSubqueries from batch of PartitionPruning") {
-+  test("SPARK-36444: Remove OptimizeSubqueries from batch of PartitionPruning",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
-       val df = sql(
-         """
-@@ -1485,7 +1501,7 @@ abstract class DynamicPartitionPruningSuiteBase
-   }
- 
-   test("SPARK-38148: Do not add dynamic partition pruning if there exists static partition " +
--    "pruning") {
-+    "pruning", IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
-       Seq(
-         "f.store_id = 1" -> false,
-@@ -1557,7 +1573,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-38674: Remove useless deduplicate in SubqueryBroadcastExec") {
-+  test("SPARK-38674: Remove useless deduplicate in SubqueryBroadcastExec",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withTable("duplicate_keys") {
-       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
-         Seq[(Int, String)]((1, "NL"), (1, "NL"), (3, "US"), (3, "US"), (3, "US"))
-@@ -1588,7 +1605,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-39338: Remove dynamic pruning subquery if pruningKey's references is empty") {
-+  test("SPARK-39338: Remove dynamic pruning subquery if pruningKey's references is empty",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
-       val df = sql(
-         """
-@@ -1617,7 +1635,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-39217: Makes DPP support the pruning side has Union") {
-+  test("SPARK-39217: Makes DPP support the pruning side has Union",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
-       val df = sql(
-         """
-@@ -1729,6 +1748,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
-               case s: BatchScanExec =>
-                 // we use f1 col for v2 tables due to schema pruning
-                 s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))
-+              case s: CometScanExec =>
-+                s.output.exists(_.exists(_.argString(maxFields = 100).contains("fid")))
-               case _ => false
-             }
-           assert(scanOption.isDefined)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-index a206e97c353..fea1149b67d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-@@ -467,7 +467,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
-     }
-   }
- 
--  test("Explain formatted output for scan operator for datasource V2") {
-+  test("Explain formatted output for scan operator for datasource V2",
-+      IgnoreComet("Comet explain output is different")) {
-     withTempDir { dir =>
-       Seq("parquet", "orc", "csv", "json").foreach { fmt =>
-         val basePath = dir.getCanonicalPath + "/" + fmt
-@@ -545,7 +546,9 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
-   }
- }
- 
--class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuite {
-+// Ignored when Comet is enabled. Comet changes expected query plans.
-+class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuite
-+    with IgnoreCometSuite {
-   import testImplicits._
- 
-   test("SPARK-35884: Explain Formatted") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-index 93275487f29..01e5c601763 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-@@ -23,6 +23,7 @@ import java.nio.file.{Files, StandardOpenOption}
- 
- import scala.collection.mutable
- 
-+import org.apache.comet.CometConf
- import org.apache.hadoop.conf.Configuration
- import org.apache.hadoop.fs.{LocalFileSystem, Path}
- 
-@@ -33,6 +34,7 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GreaterTha
- import org.apache.spark.sql.catalyst.expressions.IntegralLiteralTestUtils.{negativeInt, positiveInt}
- import org.apache.spark.sql.catalyst.plans.logical.Filter
- import org.apache.spark.sql.catalyst.types.DataTypeUtils
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec, CometScanExec, CometSortMergeJoinExec}
- import org.apache.spark.sql.execution.{FileSourceScanLike, SimpleMode}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.datasources.FilePartition
-@@ -250,6 +252,8 @@ class FileBasedDataSourceSuite extends QueryTest
-               case "" => "_LEGACY_ERROR_TEMP_2062"
-               case _ => "_LEGACY_ERROR_TEMP_2055"
-             }
-+            // native_datafusion Parquet scan cannot throw a SparkFileNotFoundException
-+            assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_DATAFUSION)
-             checkErrorMatchPVals(
-               exception = intercept[SparkException] {
-                 testIgnoreMissingFiles(options)
-@@ -955,6 +959,7 @@ class FileBasedDataSourceSuite extends QueryTest
-             assert(bJoinExec.isEmpty)
-             val smJoinExec = collect(joinedDF.queryExecution.executedPlan) {
-               case smJoin: SortMergeJoinExec => smJoin
-+              case smJoin: CometSortMergeJoinExec => smJoin
-             }
-             assert(smJoinExec.nonEmpty)
-           }
-@@ -1015,6 +1020,7 @@ class FileBasedDataSourceSuite extends QueryTest
- 
-           val fileScan = df.queryExecution.executedPlan collectFirst {
-             case BatchScanExec(_, f: FileScan, _, _, _, _) => f
-+            case CometBatchScanExec(BatchScanExec(_, f: FileScan, _, _, _, _), _) => f
-           }
-           assert(fileScan.nonEmpty)
-           assert(fileScan.get.partitionFilters.nonEmpty)
-@@ -1056,6 +1062,7 @@ class FileBasedDataSourceSuite extends QueryTest
- 
-           val fileScan = df.queryExecution.executedPlan collectFirst {
-             case BatchScanExec(_, f: FileScan, _, _, _, _) => f
-+            case CometBatchScanExec(BatchScanExec(_, f: FileScan, _, _, _, _), _) => f
-           }
-           assert(fileScan.nonEmpty)
-           assert(fileScan.get.partitionFilters.isEmpty)
-@@ -1240,6 +1247,9 @@ class FileBasedDataSourceSuite extends QueryTest
-           val filters = df.queryExecution.executedPlan.collect {
-             case f: FileSourceScanLike => f.dataFilters
-             case b: BatchScanExec => b.scan.asInstanceOf[FileScan].dataFilters
-+            case b: CometScanExec => b.dataFilters
-+            case b: CometNativeScanExec => b.dataFilters
-+            case b: CometBatchScanExec => b.scan.asInstanceOf[FileScan].dataFilters
-           }.flatten
-           assert(filters.contains(GreaterThan(scan.logicalPlan.output.head, Literal(5L))))
-         }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-new file mode 100644
-index 00000000000..5db1532cf9f
---- /dev/null
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-@@ -0,0 +1,45 @@
-+/*
-+ * Licensed to the Apache Software Foundation (ASF) under one or more
-+ * contributor license agreements.  See the NOTICE file distributed with
-+ * this work for additional information regarding copyright ownership.
-+ * The ASF licenses this file to You under the Apache License, Version 2.0
-+ * (the "License"); you may not use this file except in compliance with
-+ * the License.  You may obtain a copy of the License at
-+ *
-+ *    http://www.apache.org/licenses/LICENSE-2.0
-+ *
-+ * Unless required by applicable law or agreed to in writing, software
-+ * distributed under the License is distributed on an "AS IS" BASIS,
-+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ * See the License for the specific language governing permissions and
-+ * limitations under the License.
-+ */
-+
-+package org.apache.spark.sql
-+
-+import org.scalactic.source.Position
-+import org.scalatest.Tag
-+
-+import org.apache.spark.sql.test.SQLTestUtils
-+
-+/**
-+ * Tests with this tag will be ignored when Comet is enabled (e.g., via `ENABLE_COMET`).
-+ */
-+case class IgnoreComet(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeIcebergCompat(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeDataFusion(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeScan(reason: String) extends Tag("DisableComet")
-+
-+/**
-+ * Helper trait that disables Comet for all tests regardless of default config values.
-+ */
-+trait IgnoreCometSuite extends SQLTestUtils {
-+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
-+    (implicit pos: Position): Unit = {
-+    if (isCometEnabled) {
-+      ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
-+    } else {
-+      super.test(testName, testTags: _*)(testFun)
-+    }
-+  }
-+}
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala
-index fedfd9ff587..c5bfc8f16e4 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala
-@@ -505,7 +505,8 @@ class InjectRuntimeFilterSuite extends QueryTest with SQLTestUtils with SharedSp
-   }
- 
-   test("Runtime bloom filter join: do not add bloom filter if dpp filter exists " +
--    "on the same column") {
-+    "on the same column",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.RUNTIME_BLOOM_FILTER_APPLICATION_SIDE_SCAN_SIZE_THRESHOLD.key -> "3000",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "2000") {
-       assertDidNotRewriteWithBloomFilter("select * from bf5part join bf2 on " +
-@@ -514,7 +515,8 @@ class InjectRuntimeFilterSuite extends QueryTest with SQLTestUtils with SharedSp
-   }
- 
-   test("Runtime bloom filter join: add bloom filter if dpp filter exists on " +
--    "a different column") {
-+    "a different column",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     withSQLConf(SQLConf.RUNTIME_BLOOM_FILTER_APPLICATION_SIDE_SCAN_SIZE_THRESHOLD.key -> "3000",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "2000") {
-       assertRewroteWithBloomFilter("select * from bf5part join bf2 on " +
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
-index 7af826583bd..3c3def1eb67 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
-@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide
- import org.apache.spark.sql.catalyst.plans.PlanTest
- import org.apache.spark.sql.catalyst.plans.logical._
- import org.apache.spark.sql.catalyst.rules.RuleExecutor
-+import org.apache.spark.sql.comet.{CometHashJoinExec, CometSortMergeJoinExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.joins._
- import org.apache.spark.sql.internal.SQLConf
-@@ -362,6 +363,7 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
-     val executedPlan = df.queryExecution.executedPlan
-     val shuffleHashJoins = collect(executedPlan) {
-       case s: ShuffledHashJoinExec => s
-+      case c: CometHashJoinExec => c.originalPlan.asInstanceOf[ShuffledHashJoinExec]
-     }
-     assert(shuffleHashJoins.size == 1)
-     assert(shuffleHashJoins.head.buildSide == buildSide)
-@@ -371,6 +373,7 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
-     val executedPlan = df.queryExecution.executedPlan
-     val shuffleMergeJoins = collect(executedPlan) {
-       case s: SortMergeJoinExec => s
-+      case c: CometSortMergeJoinExec => c
-     }
-     assert(shuffleMergeJoins.size == 1)
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-index 4d256154c85..66a5473852d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-@@ -31,7 +31,8 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
- import org.apache.spark.sql.catalyst.expressions.{Ascending, GenericRow, SortOrder}
- import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
- import org.apache.spark.sql.catalyst.plans.logical.{Filter, HintInfo, Join, JoinHint, NO_BROADCAST_AND_REPLICATION}
--import org.apache.spark.sql.execution.{BinaryExecNode, FilterExec, ProjectExec, SortExec, SparkPlan, WholeStageCodegenExec}
-+import org.apache.spark.sql.comet._
-+import org.apache.spark.sql.execution.{BinaryExecNode, ColumnarToRowExec, FilterExec, InputAdapter, ProjectExec, SortExec, SparkPlan, WholeStageCodegenExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.joins._
-@@ -802,7 +803,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-     }
-   }
- 
--  test("test SortMergeJoin (with spill)") {
-+  test("test SortMergeJoin (with spill)",
-+      IgnoreComet("TODO: Comet SMJ doesn't support spill yet")) {
-     withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1",
-       SQLConf.SORT_MERGE_JOIN_EXEC_BUFFER_IN_MEMORY_THRESHOLD.key -> "0",
-       SQLConf.SORT_MERGE_JOIN_EXEC_BUFFER_SPILL_THRESHOLD.key -> "1") {
-@@ -928,10 +930,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-       val physical = df.queryExecution.sparkPlan
-       val physicalJoins = physical.collect {
-         case j: SortMergeJoinExec => j
-+        case j: CometSortMergeJoinExec => j.originalPlan.asInstanceOf[SortMergeJoinExec]
-       }
-       val executed = df.queryExecution.executedPlan
-       val executedJoins = collect(executed) {
-         case j: SortMergeJoinExec => j
-+        case j: CometSortMergeJoinExec => j.originalPlan.asInstanceOf[SortMergeJoinExec]
-       }
-       // This only applies to the above tested queries, in which a child SortMergeJoin always
-       // contains the SortOrder required by its parent SortMergeJoin. Thus, SortExec should never
-@@ -1177,9 +1181,11 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-       val plan = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2", joinType)
-         .groupBy($"k1").count()
-         .queryExecution.executedPlan
--      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: ShuffledHashJoinExec | _: CometHashJoinExec => true }.size === 1)
-       // No extra shuffle before aggregate
--      assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 2)
-+      assert(collect(plan) {
-+        case _: ShuffleExchangeLike => true }.size === 2)
-     })
-   }
- 
-@@ -1196,10 +1202,11 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-         .join(df4.hint("SHUFFLE_MERGE"), $"k1" === $"k4", joinType)
-         .queryExecution
-         .executedPlan
--      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 2)
-+      assert(collect(plan) {
-+        case _: SortMergeJoinExec | _: CometSortMergeJoinExec => true }.size === 2)
-       assert(collect(plan) { case _: BroadcastHashJoinExec => true }.size === 1)
-       // No extra sort before last sort merge join
--      assert(collect(plan) { case _: SortExec => true }.size === 3)
-+      assert(collect(plan) { case _: SortExec | _: CometSortExec => true }.size === 3)
-     })
- 
-     // Test shuffled hash join
-@@ -1209,10 +1216,13 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-         .join(df4.hint("SHUFFLE_MERGE"), $"k1" === $"k4", joinType)
-         .queryExecution
-         .executedPlan
--      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 2)
--      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: SortMergeJoinExec | _: CometSortMergeJoinExec => true }.size === 2)
-+      assert(collect(plan) {
-+        case _: ShuffledHashJoinExec | _: CometHashJoinExec => true }.size === 1)
-       // No extra sort before last sort merge join
--      assert(collect(plan) { case _: SortExec => true }.size === 3)
-+      assert(collect(plan) {
-+        case _: SortExec | _: CometSortExec => true }.size === 3)
-     })
-   }
- 
-@@ -1303,12 +1313,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-     inputDFs.foreach { case (df1, df2, joinExprs) =>
-       val smjDF = df1.join(df2.hint("SHUFFLE_MERGE"), joinExprs, "full")
-       assert(collect(smjDF.queryExecution.executedPlan) {
--        case _: SortMergeJoinExec => true }.size === 1)
-+        case _: SortMergeJoinExec | _: CometSortMergeJoinExec => true }.size === 1)
-       val smjResult = smjDF.collect()
- 
-       val shjDF = df1.join(df2.hint("SHUFFLE_HASH"), joinExprs, "full")
-       assert(collect(shjDF.queryExecution.executedPlan) {
--        case _: ShuffledHashJoinExec => true }.size === 1)
-+        case _: ShuffledHashJoinExec | _: CometHashJoinExec => true }.size === 1)
-       // Same result between shuffled hash join and sort merge join
-       checkAnswer(shjDF, smjResult)
-     }
-@@ -1367,12 +1377,14 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val smjDF = df1.hint("SHUFFLE_MERGE").join(df2, joinExprs, "leftouter")
-           assert(collect(smjDF.queryExecution.executedPlan) {
-             case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-           }.size === 1)
-           val smjResult = smjDF.collect()
- 
-           val shjDF = df1.hint("SHUFFLE_HASH").join(df2, joinExprs, "leftouter")
-           assert(collect(shjDF.queryExecution.executedPlan) {
-             case _: ShuffledHashJoinExec => true
-+            case _: CometHashJoinExec => true
-           }.size === 1)
-           // Same result between shuffled hash join and sort merge join
-           checkAnswer(shjDF, smjResult)
-@@ -1383,12 +1395,14 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val smjDF = df2.join(df1.hint("SHUFFLE_MERGE"), joinExprs, "rightouter")
-           assert(collect(smjDF.queryExecution.executedPlan) {
-             case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-           }.size === 1)
-           val smjResult = smjDF.collect()
- 
-           val shjDF = df2.join(df1.hint("SHUFFLE_HASH"), joinExprs, "rightouter")
-           assert(collect(shjDF.queryExecution.executedPlan) {
-             case _: ShuffledHashJoinExec => true
-+            case _: CometHashJoinExec => true
-           }.size === 1)
-           // Same result between shuffled hash join and sort merge join
-           checkAnswer(shjDF, smjResult)
-@@ -1432,13 +1446,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-         assert(shjCodegenDF.queryExecution.executedPlan.collect {
-           case WholeStageCodegenExec(_ : ShuffledHashJoinExec) => true
-           case WholeStageCodegenExec(ProjectExec(_, _ : ShuffledHashJoinExec)) => true
-+          case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(_: CometHashJoinExec))) =>
-+            true
-+          case WholeStageCodegenExec(ColumnarToRowExec(
-+            InputAdapter(CometProjectExec(_, _, _, _, _: CometHashJoinExec, _)))) => true
-+          case _: CometHashJoinExec => true
-         }.size === 1)
-         checkAnswer(shjCodegenDF, Seq.empty)
- 
-         withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
-           val shjNonCodegenDF = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2", joinType)
-           assert(shjNonCodegenDF.queryExecution.executedPlan.collect {
--            case _: ShuffledHashJoinExec => true }.size === 1)
-+            case _: ShuffledHashJoinExec => true
-+            case _: CometHashJoinExec => true
-+          }.size === 1)
-           checkAnswer(shjNonCodegenDF, Seq.empty)
-         }
-       }
-@@ -1486,7 +1507,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
-           assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
-           // Have shuffle before aggregation
--          assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 1)
-+          assert(collect(plan) {
-+            case _: ShuffleExchangeLike => true }.size === 1)
-       }
- 
-       def getJoinQuery(selectExpr: String, joinType: String): String = {
-@@ -1515,9 +1537,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           }
-           val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
-           assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
--          assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
-+          assert(collect(plan) {
-+            case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-+          }.size === 3)
-           // No extra sort on left side before last sort merge join
--          assert(collect(plan) { case _: SortExec => true }.size === 5)
-+          assert(collect(plan) { case _: SortExec | _: CometSortExec => true }.size === 5)
-       }
- 
-       // Test output ordering is not preserved
-@@ -1526,9 +1551,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val selectExpr = "/*+ BROADCAST(left_t) */ k1 as k0"
-           val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
-           assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
--          assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
-+          assert(collect(plan) {
-+            case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-+          }.size === 3)
-           // Have sort on left side before last sort merge join
--          assert(collect(plan) { case _: SortExec => true }.size === 6)
-+          assert(collect(plan) { case _: SortExec | _: CometSortExec => true }.size === 6)
-       }
- 
-       // Test singe partition
-@@ -1538,7 +1566,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-            |FROM range(0, 10, 1, 1) t1 FULL OUTER JOIN range(0, 10, 1, 1) t2
-            |""".stripMargin)
-       val plan = fullJoinDF.queryExecution.executedPlan
--      assert(collect(plan) { case _: ShuffleExchangeExec => true}.size == 1)
-+      assert(collect(plan) {
-+        case _: ShuffleExchangeLike => true}.size == 1)
-       checkAnswer(fullJoinDF, Row(100))
-     }
-   }
-@@ -1583,6 +1612,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           Seq(semiJoinDF, antiJoinDF).foreach { df =>
-             assert(collect(df.queryExecution.executedPlan) {
-               case j: ShuffledHashJoinExec if j.ignoreDuplicatedKey == ignoreDuplicatedKey => true
-+              case j: CometHashJoinExec
-+                if j.originalPlan.asInstanceOf[ShuffledHashJoinExec].ignoreDuplicatedKey ==
-+                  ignoreDuplicatedKey => true
-             }.size == 1)
-           }
-       }
-@@ -1627,14 +1659,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
- 
-   test("SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)") {
-     def check(plan: SparkPlan): Unit = {
--      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: SortMergeJoinExec => true
-+        case _: CometSortMergeJoinExec => true
-+      }.size === 1)
-     }
-     dupStreamSideColTest("MERGE", check)
-   }
- 
-   test("SPARK-43113: Full outer join with duplicate stream-side references in condition (SHJ)") {
-     def check(plan: SparkPlan): Unit = {
--      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: ShuffledHashJoinExec => true
-+        case _: CometHashJoinExec => true
-+      }.size === 1)
-     }
-     dupStreamSideColTest("SHUFFLE_HASH", check)
-   }
-@@ -1770,7 +1808,8 @@ class ThreadLeakInSortMergeJoinSuite
-       sparkConf.set(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD, 20))
-   }
- 
--  test("SPARK-47146: thread leak when doing SortMergeJoin (with spill)") {
-+  test("SPARK-47146: thread leak when doing SortMergeJoin (with spill)",
-+    IgnoreComet("Comet does not support spilling")) {
- 
-     withSQLConf(
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
-index c26757c9cff..d55775f09d7 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
-@@ -69,7 +69,7 @@ import org.apache.spark.tags.ExtendedSQLTest
-  * }}}
-  */
- // scalastyle:on line.size.limit
--trait PlanStabilitySuite extends DisableAdaptiveExecutionSuite {
-+trait PlanStabilitySuite extends DisableAdaptiveExecutionSuite with IgnoreCometSuite {
- 
-   protected val baseResourcePath = {
-     // use the same way as `SQLQueryTestSuite` to get the resource path
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
-index 793a0da6a86..6ccb9d62582 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
-@@ -1521,7 +1521,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
-     checkAnswer(sql("select -0.001"), Row(BigDecimal("-0.001")))
-   }
- 
--  test("external sorting updates peak execution memory") {
-+  test("external sorting updates peak execution memory",
-+    IgnoreComet("TODO: native CometSort does not update peak execution memory")) {
-     AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") {
-       sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect()
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-index 8b4ac474f87..3f79f20822f 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-@@ -223,6 +223,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     withSession(extensions) { session =>
-       session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true)
-       session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1")
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      session.conf.set("spark.comet.enabled", false)
-       assert(session.sessionState.columnarRules.contains(
-         MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
-       import session.sqlContext.implicits._
-@@ -281,6 +283,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     }
-     withSession(extensions) { session =>
-       session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE)
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      session.conf.set("spark.comet.enabled", false)
-       assert(session.sessionState.columnarRules.contains(
-         MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
-       import session.sqlContext.implicits._
-@@ -319,6 +323,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     val session = SparkSession.builder()
-       .master("local[1]")
-       .config(COLUMN_BATCH_SIZE.key, 2)
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      .config("spark.comet.enabled", false)
-       .withExtensions { extensions =>
-         extensions.injectColumnar(session =>
-           MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
-index fa1a64460fc..1d2e215d6a3 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
-@@ -17,6 +17,8 @@
- 
- package org.apache.spark.sql
- 
-+import org.apache.comet.CometConf
-+
- import org.apache.spark.{SPARK_DOC_ROOT, SparkIllegalArgumentException, SparkRuntimeException}
- import org.apache.spark.sql.catalyst.expressions.Cast._
- import org.apache.spark.sql.execution.FormattedMode
-@@ -178,29 +180,31 @@ class StringFunctionsSuite extends QueryTest with SharedSparkSession {
-   }
- 
-   test("string regex_replace / regex_extract") {
--    val df = Seq(
--      ("100-200", "(\\d+)-(\\d+)", "300"),
--      ("100-200", "(\\d+)-(\\d+)", "400"),
--      ("100-200", "(\\d+)", "400")).toDF("a", "b", "c")
-+    withSQLConf(CometConf.COMET_REGEXP_ALLOW_INCOMPATIBLE.key -> "true") {
-+      val df = Seq(
-+        ("100-200", "(\\d+)-(\\d+)", "300"),
-+        ("100-200", "(\\d+)-(\\d+)", "400"),
-+        ("100-200", "(\\d+)", "400")).toDF("a", "b", "c")
- 
--    checkAnswer(
--      df.select(
--        regexp_replace($"a", "(\\d+)", "num"),
--        regexp_replace($"a", $"b", $"c"),
--        regexp_extract($"a", "(\\d+)-(\\d+)", 1)),
--      Row("num-num", "300", "100") :: Row("num-num", "400", "100") ::
--        Row("num-num", "400-400", "100") :: Nil)
--
--    // for testing the mutable state of the expression in code gen.
--    // This is a hack way to enable the codegen, thus the codegen is enable by default,
--    // it will still use the interpretProjection if projection followed by a LocalRelation,
--    // hence we add a filter operator.
--    // See the optimizer rule `ConvertToLocalRelation`
--    checkAnswer(
--      df.filter("isnotnull(a)").selectExpr(
--        "regexp_replace(a, b, c)",
--        "regexp_extract(a, b, 1)"),
--      Row("300", "100") :: Row("400", "100") :: Row("400-400", "100") :: Nil)
-+      checkAnswer(
-+        df.select(
-+          regexp_replace($"a", "(\\d+)", "num"),
-+          regexp_replace($"a", $"b", $"c"),
-+          regexp_extract($"a", "(\\d+)-(\\d+)", 1)),
-+        Row("num-num", "300", "100") :: Row("num-num", "400", "100") ::
-+          Row("num-num", "400-400", "100") :: Nil)
-+
-+      // for testing the mutable state of the expression in code gen.
-+      // This is a hack way to enable the codegen, thus the codegen is enable by default,
-+      // it will still use the interpretProjection if projection followed by a LocalRelation,
-+      // hence we add a filter operator.
-+      // See the optimizer rule `ConvertToLocalRelation`
-+      checkAnswer(
-+        df.filter("isnotnull(a)").selectExpr(
-+          "regexp_replace(a, b, c)",
-+          "regexp_extract(a, b, 1)"),
-+        Row("300", "100") :: Row("400", "100") :: Row("400-400", "100") :: Nil)
-+    }
-   }
- 
-   test("non-matching optional group") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-index 260c992f1ae..6297c071b19 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-@@ -22,10 +22,11 @@ import scala.collection.mutable.ArrayBuffer
- import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
- import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
- import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Join, LogicalPlan, Project, Sort, Union}
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
- import org.apache.spark.sql.execution.datasources.FileScanRDD
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, BroadcastNestedLoopJoinExec}
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
-@@ -1599,6 +1600,12 @@ class SubquerySuite extends QueryTest
-             fs.inputRDDs().forall(
-               _.asInstanceOf[FileScanRDD].filePartitions.forall(
-                 _.files.forall(_.urlEncodedPath.contains("p=0"))))
-+        case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(
-+        fs @ CometScanExec(_, _, _, _, partitionFilters, _, _, _, _, _, _)))) =>
-+          partitionFilters.exists(ExecSubqueryExpression.hasSubquery) &&
-+            fs.inputRDDs().forall(
-+              _.asInstanceOf[FileScanRDD].filePartitions.forall(
-+                _.files.forall(_.urlEncodedPath.contains("p=0"))))
-         case _ => false
-       })
-     }
-@@ -2164,7 +2171,7 @@ class SubquerySuite extends QueryTest
- 
-       df.collect()
-       val exchanges = collect(df.queryExecution.executedPlan) {
--        case s: ShuffleExchangeExec => s
-+        case s: ShuffleExchangeLike => s
-       }
-       assert(exchanges.size === 1)
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
-index d269290e616..13726a31e07 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
-@@ -24,6 +24,7 @@ import test.org.apache.spark.sql.connector._
- 
- import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
- import org.apache.spark.sql.catalyst.InternalRow
-+import org.apache.spark.sql.comet.CometSortExec
- import org.apache.spark.sql.connector.catalog.{PartitionInternalRow, SupportsRead, Table, TableCapability, TableProvider}
- import org.apache.spark.sql.connector.catalog.TableCapability._
- import org.apache.spark.sql.connector.expressions.{Expression, FieldReference, Literal, NamedReference, NullOrdering, SortDirection, SortOrder, Transform}
-@@ -34,7 +35,7 @@ import org.apache.spark.sql.connector.read.partitioning.{KeyGroupedPartitioning,
- import org.apache.spark.sql.execution.SortExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV2Relation, DataSourceV2ScanRelation}
--import org.apache.spark.sql.execution.exchange.{Exchange, ShuffleExchangeExec}
-+import org.apache.spark.sql.execution.exchange.{Exchange, ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
- import org.apache.spark.sql.expressions.Window
- import org.apache.spark.sql.functions._
-@@ -269,13 +270,13 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
-           val groupByColJ = df.groupBy($"j").agg(sum($"i"))
-           checkAnswer(groupByColJ, Seq(Row(2, 8), Row(4, 2), Row(6, 5)))
-           assert(collectFirst(groupByColJ.queryExecution.executedPlan) {
--            case e: ShuffleExchangeExec => e
-+            case e: ShuffleExchangeLike => e
-           }.isDefined)
- 
-           val groupByIPlusJ = df.groupBy($"i" + $"j").agg(count("*"))
-           checkAnswer(groupByIPlusJ, Seq(Row(5, 2), Row(6, 2), Row(8, 1), Row(9, 1)))
-           assert(collectFirst(groupByIPlusJ.queryExecution.executedPlan) {
--            case e: ShuffleExchangeExec => e
-+            case e: ShuffleExchangeLike => e
-           }.isDefined)
-         }
-       }
-@@ -335,10 +336,11 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
- 
-                 val (shuffleExpected, sortExpected) = groupByExpects
-                 assert(collectFirst(groupBy.queryExecution.executedPlan) {
--                  case e: ShuffleExchangeExec => e
-+                  case e: ShuffleExchangeLike => e
-                 }.isDefined === shuffleExpected)
-                 assert(collectFirst(groupBy.queryExecution.executedPlan) {
-                   case e: SortExec => e
-+                  case c: CometSortExec => c
-                 }.isDefined === sortExpected)
-               }
- 
-@@ -353,10 +355,11 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
- 
-                 val (shuffleExpected, sortExpected) = windowFuncExpects
-                 assert(collectFirst(windowPartByColIOrderByColJ.queryExecution.executedPlan) {
--                  case e: ShuffleExchangeExec => e
-+                  case e: ShuffleExchangeLike => e
-                 }.isDefined === shuffleExpected)
-                 assert(collectFirst(windowPartByColIOrderByColJ.queryExecution.executedPlan) {
-                   case e: SortExec => e
-+                  case c: CometSortExec => c
-                 }.isDefined === sortExpected)
-               }
-             }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-index cfc8b2cc845..c6fcfd7bd08 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-@@ -21,6 +21,7 @@ import scala.collection.mutable.ArrayBuffer
- import org.apache.spark.SparkConf
- import org.apache.spark.sql.{AnalysisException, QueryTest}
- import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability}
- import org.apache.spark.sql.connector.read.ScanBuilder
- import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
-@@ -184,7 +185,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest with SharedSparkSession {
-             val df = spark.read.format(format).load(path.getCanonicalPath)
-             checkAnswer(df, inputData.toDF())
-             assert(
--              df.queryExecution.executedPlan.exists(_.isInstanceOf[FileSourceScanExec]))
-+              df.queryExecution.executedPlan.exists {
-+                case _: FileSourceScanExec | _: CometScanExec => true
-+                case _ => false
-+              }
-+            )
-           }
-         } finally {
-           spark.listenerManager.unregister(listener)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-index 71e030f535e..d5ae6cbf3d5 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-@@ -22,6 +22,7 @@ import org.apache.spark.sql.{DataFrame, Row}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{Literal, TransformExpression}
- import org.apache.spark.sql.catalyst.plans.physical
-+import org.apache.spark.sql.comet.CometSortMergeJoinExec
- import org.apache.spark.sql.connector.catalog.Identifier
- import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog
- import org.apache.spark.sql.connector.catalog.functions._
-@@ -31,7 +32,7 @@ import org.apache.spark.sql.connector.expressions.Expressions._
- import org.apache.spark.sql.execution.SparkPlan
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.joins.SortMergeJoinExec
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.internal.SQLConf._
-@@ -282,13 +283,14 @@ class KeyGroupedPartitioningSuite extends DistributionAndOrderingSuiteBase {
-         Row("bbb", 20, 250.0), Row("bbb", 20, 350.0), Row("ccc", 30, 400.50)))
-   }
- 
--  private def collectShuffles(plan: SparkPlan): Seq[ShuffleExchangeExec] = {
-+  private def collectShuffles(plan: SparkPlan): Seq[ShuffleExchangeLike] = {
-     // here we skip collecting shuffle operators that are not associated with SMJ
-     collect(plan) {
-       case s: SortMergeJoinExec => s
-+      case c: CometSortMergeJoinExec => c.originalPlan
-     }.flatMap(smj =>
-       collect(smj) {
--        case s: ShuffleExchangeExec => s
-+        case s: ShuffleExchangeLike => s
-       })
-   }
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
-index 40938eb6424..fad0fc1e1f0 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
-@@ -21,7 +21,7 @@ package org.apache.spark.sql.connector
- import java.sql.Date
- import java.util.Collections
- 
--import org.apache.spark.sql.{catalyst, AnalysisException, DataFrame, Row}
-+import org.apache.spark.sql.{catalyst, AnalysisException, DataFrame, IgnoreCometSuite, Row}
- import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression, Cast, Literal}
- import org.apache.spark.sql.catalyst.expressions.objects.Invoke
- import org.apache.spark.sql.catalyst.plans.physical
-@@ -45,7 +45,8 @@ import org.apache.spark.sql.util.QueryExecutionListener
- import org.apache.spark.tags.SlowSQLTest
- 
- @SlowSQLTest
--class WriteDistributionAndOrderingSuite extends DistributionAndOrderingSuiteBase {
-+class WriteDistributionAndOrderingSuite extends DistributionAndOrderingSuiteBase
-+  with IgnoreCometSuite {
-   import testImplicits._
- 
-   before {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
-index ae1c0a86a14..1d3b914fd64 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
-@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.permission.FsPermission
- import org.mockito.Mockito.{mock, spy, when}
- 
- import org.apache.spark._
--import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, QueryTest, Row, SaveMode}
-+import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, IgnoreComet, QueryTest, Row, SaveMode}
- import org.apache.spark.sql.catalyst.FunctionIdentifier
- import org.apache.spark.sql.catalyst.analysis.{NamedParameter, UnresolvedGenerator}
- import org.apache.spark.sql.catalyst.expressions.{Grouping, Literal, RowNumber}
-@@ -256,7 +256,8 @@ class QueryExecutionErrorsSuite
-   }
- 
-   test("INCONSISTENT_BEHAVIOR_CROSS_VERSION: " +
--    "compatibility with Spark 2.4/3.2 in reading/writing dates") {
-+    "compatibility with Spark 2.4/3.2 in reading/writing dates",
-+    IgnoreComet("Comet doesn't completely support datetime rebase mode yet")) {
- 
-     // Fail to read ancient datetime values.
-     withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_READ.key -> EXCEPTION.toString) {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-index 418ca3430bb..eb8267192f8 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-@@ -23,7 +23,7 @@ import scala.util.Random
- import org.apache.hadoop.fs.Path
- 
- import org.apache.spark.SparkConf
--import org.apache.spark.sql.{DataFrame, QueryTest}
-+import org.apache.spark.sql.{DataFrame, IgnoreComet, QueryTest}
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
- import org.apache.spark.sql.internal.SQLConf
-@@ -195,7 +195,7 @@ class DataSourceV2ScanExecRedactionSuite extends DataSourceScanRedactionTest {
-     }
-   }
- 
--  test("FileScan description") {
-+  test("FileScan description", IgnoreComet("Comet doesn't use BatchScan")) {
-     Seq("json", "orc", "parquet").foreach { format =>
-       withTempPath { path =>
-         val dir = path.getCanonicalPath
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
-index 743ec41dbe7..9f30d6c8e04 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
-@@ -53,6 +53,10 @@ class LogicalPlanTagInSparkPlanSuite extends TPCDSQuerySuite with DisableAdaptiv
-     case ColumnarToRowExec(i: InputAdapter) => isScanPlanTree(i.child)
-     case p: ProjectExec => isScanPlanTree(p.child)
-     case f: FilterExec => isScanPlanTree(f.child)
-+    // Comet produces scan plan tree like:
-+    // ColumnarToRow
-+    //  +- ReusedExchange
-+    case _: ReusedExchangeExec => false
-     case _: LeafExecNode => true
-     case _ => false
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
-index de24b8c82b0..1f835481290 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.rdd.RDD
--import org.apache.spark.sql.{execution, DataFrame, Row}
-+import org.apache.spark.sql.{execution, DataFrame, IgnoreCometSuite, Row}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions._
- import org.apache.spark.sql.catalyst.plans._
-@@ -35,7 +35,9 @@ import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.sql.types._
- 
--class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
-+// Ignore this suite when Comet is enabled. This suite tests the Spark planner and Comet planner
-+// comes out with too many difference. Simply ignoring this suite for now.
-+class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper with IgnoreCometSuite {
-   import testImplicits._
- 
-   setupTestData()
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
-index 9e9d717db3b..c1a7caf56e0 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
-@@ -17,7 +17,8 @@
- 
- package org.apache.spark.sql.execution
- 
--import org.apache.spark.sql.{DataFrame, QueryTest, Row}
-+import org.apache.spark.sql.{DataFrame, IgnoreComet, QueryTest, Row}
-+import org.apache.spark.sql.comet.CometProjectExec
- import org.apache.spark.sql.connector.SimpleWritableDataSource
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.internal.SQLConf
-@@ -34,7 +35,10 @@ abstract class RemoveRedundantProjectsSuiteBase
-   private def assertProjectExecCount(df: DataFrame, expected: Int): Unit = {
-     withClue(df.queryExecution) {
-       val plan = df.queryExecution.executedPlan
--      val actual = collectWithSubqueries(plan) { case p: ProjectExec => p }.size
-+      val actual = collectWithSubqueries(plan) {
-+        case p: ProjectExec => p
-+        case p: CometProjectExec => p
-+      }.size
-       assert(actual == expected)
-     }
-   }
-@@ -112,7 +116,8 @@ abstract class RemoveRedundantProjectsSuiteBase
-     assertProjectExec(query, 1, 3)
-   }
- 
--  test("join with ordering requirement") {
-+  test("join with ordering requirement",
-+    IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) {
-     val query = "select * from (select key, a, c, b from testView) as t1 join " +
-       "(select key, a, b, c from testView) as t2 on t1.key = t2.key where t2.a > 50"
-     assertProjectExec(query, 2, 2)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
-index 005e764cc30..92ec088efab 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
-@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
- 
- import org.apache.spark.sql.{DataFrame, QueryTest}
- import org.apache.spark.sql.catalyst.plans.physical.{RangePartitioning, UnknownPartitioning}
-+import org.apache.spark.sql.comet.CometSortExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.execution.joins.ShuffledJoin
- import org.apache.spark.sql.internal.SQLConf
-@@ -33,7 +34,7 @@ abstract class RemoveRedundantSortsSuiteBase
- 
-   private def checkNumSorts(df: DataFrame, count: Int): Unit = {
-     val plan = df.queryExecution.executedPlan
--    assert(collectWithSubqueries(plan) { case s: SortExec => s }.length == count)
-+    assert(collectWithSubqueries(plan) { case _: SortExec | _: CometSortExec => 1 }.length == count)
-   }
- 
-   private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
-index 47679ed7865..9ffbaecb98e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.sql.{DataFrame, QueryTest}
-+import org.apache.spark.sql.comet.CometHashAggregateExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
- import org.apache.spark.sql.internal.SQLConf
-@@ -31,7 +32,7 @@ abstract class ReplaceHashWithSortAggSuiteBase
-   private def checkNumAggs(df: DataFrame, hashAggCount: Int, sortAggCount: Int): Unit = {
-     val plan = df.queryExecution.executedPlan
-     assert(collectWithSubqueries(plan) {
--      case s @ (_: HashAggregateExec | _: ObjectHashAggregateExec) => s
-+      case s @ (_: HashAggregateExec | _: ObjectHashAggregateExec | _: CometHashAggregateExec ) => s
-     }.length == hashAggCount)
-     assert(collectWithSubqueries(plan) { case s: SortAggregateExec => s }.length == sortAggCount)
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
-index eec396b2e39..bf3f1c769d6 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.TestUtils.assertSpilled
--import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, QueryTest, Row}
- import org.apache.spark.sql.internal.SQLConf.{WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD, WINDOW_EXEC_BUFFER_SPILL_THRESHOLD}
- import org.apache.spark.sql.test.SharedSparkSession
- 
-@@ -470,7 +470,7 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
-       Row(1, 3, null) :: Row(2, null, 4) :: Nil)
-   }
- 
--  test("test with low buffer spill threshold") {
-+  test("test with low buffer spill threshold", IgnoreComet("Comet does not support spilling")) {
-     val nums = sparkContext.parallelize(1 to 10).map(x => (x, x % 2)).toDF("x", "y")
-     nums.createOrReplaceTempView("nums")
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
-index b14f4a405f6..ab7baf434a5 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
-@@ -23,6 +23,7 @@ import org.apache.spark.sql.QueryTest
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
- import org.apache.spark.sql.catalyst.plans.logical.Deduplicate
-+import org.apache.spark.sql.comet.CometColumnarToRowExec
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
-@@ -131,7 +132,10 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession {
-         spark.range(1).write.parquet(path.getAbsolutePath)
-         val df = spark.read.parquet(path.getAbsolutePath)
-         val columnarToRowExec =
--          df.queryExecution.executedPlan.collectFirst { case p: ColumnarToRowExec => p }.get
-+          df.queryExecution.executedPlan.collectFirst {
-+            case p: ColumnarToRowExec => p
-+            case p: CometColumnarToRowExec => p
-+          }.get
-         try {
-           spark.range(1).foreach { _ =>
-             columnarToRowExec.canonicalized
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
-index 5a413c77754..207b66e1d7b 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
-@@ -17,7 +17,7 @@
- 
- package org.apache.spark.sql.execution
- 
--import org.apache.spark.sql.{Dataset, QueryTest, Row, SaveMode}
-+import org.apache.spark.sql.{Dataset, IgnoreCometSuite, QueryTest, Row, SaveMode}
- import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
- import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeAndComment, CodeGenerator}
- import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
-@@ -30,7 +30,7 @@ import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
- 
- // Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
--class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
-+class WholeStageCodegenSuite extends QueryTest with SharedSparkSession with IgnoreCometSuite
-   with DisableAdaptiveExecutionSuite {
- 
-   import testImplicits._
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
-index 2f8e401e743..a4f94417dcc 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
-@@ -27,9 +27,11 @@ import org.scalatest.time.SpanSugar._
- import org.apache.spark.SparkException
- import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
- import org.apache.spark.shuffle.sort.SortShuffleManager
--import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession, Strategy}
-+import org.apache.spark.sql.{Dataset, IgnoreComet, QueryTest, Row, SparkSession, Strategy}
- import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
- import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
-+import org.apache.spark.sql.comet._
-+import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.aggregate.BaseAggregateExec
- import org.apache.spark.sql.execution.columnar.{InMemoryTableScanExec, InMemoryTableScanLike}
-@@ -117,6 +119,7 @@ class AdaptiveQueryExecSuite
-   private def findTopLevelBroadcastHashJoin(plan: SparkPlan): Seq[BroadcastHashJoinExec] = {
-     collect(plan) {
-       case j: BroadcastHashJoinExec => j
-+      case j: CometBroadcastHashJoinExec => j.originalPlan.asInstanceOf[BroadcastHashJoinExec]
-     }
-   }
- 
-@@ -129,36 +132,46 @@ class AdaptiveQueryExecSuite
-   private def findTopLevelSortMergeJoin(plan: SparkPlan): Seq[SortMergeJoinExec] = {
-     collect(plan) {
-       case j: SortMergeJoinExec => j
-+      case j: CometSortMergeJoinExec =>
-+        assert(j.originalPlan.isInstanceOf[SortMergeJoinExec])
-+        j.originalPlan.asInstanceOf[SortMergeJoinExec]
-     }
-   }
- 
-   private def findTopLevelShuffledHashJoin(plan: SparkPlan): Seq[ShuffledHashJoinExec] = {
-     collect(plan) {
-       case j: ShuffledHashJoinExec => j
-+      case j: CometHashJoinExec => j.originalPlan.asInstanceOf[ShuffledHashJoinExec]
-     }
-   }
- 
-   private def findTopLevelBaseJoin(plan: SparkPlan): Seq[BaseJoinExec] = {
-     collect(plan) {
-       case j: BaseJoinExec => j
-+      case c: CometHashJoinExec => c.originalPlan.asInstanceOf[BaseJoinExec]
-+      case c: CometSortMergeJoinExec => c.originalPlan.asInstanceOf[BaseJoinExec]
-+      case c: CometBroadcastHashJoinExec => c.originalPlan.asInstanceOf[BaseJoinExec]
-     }
-   }
- 
-   private def findTopLevelSort(plan: SparkPlan): Seq[SortExec] = {
-     collect(plan) {
-       case s: SortExec => s
-+      case s: CometSortExec => s.originalPlan.asInstanceOf[SortExec]
-     }
-   }
- 
-   private def findTopLevelAggregate(plan: SparkPlan): Seq[BaseAggregateExec] = {
-     collect(plan) {
-       case agg: BaseAggregateExec => agg
-+      case agg: CometHashAggregateExec => agg.originalPlan.asInstanceOf[BaseAggregateExec]
-     }
-   }
- 
-   private def findTopLevelLimit(plan: SparkPlan): Seq[CollectLimitExec] = {
-     collect(plan) {
-       case l: CollectLimitExec => l
-+      case l: CometCollectLimitExec => l.originalPlan.asInstanceOf[CollectLimitExec]
-     }
-   }
- 
-@@ -202,6 +215,7 @@ class AdaptiveQueryExecSuite
-       val parts = rdd.partitions
-       assert(parts.forall(rdd.preferredLocations(_).nonEmpty))
-     }
-+
-     assert(numShuffles === (numLocalReads.length + numShufflesWithoutLocalRead))
-   }
- 
-@@ -210,7 +224,7 @@ class AdaptiveQueryExecSuite
-     val plan = df.queryExecution.executedPlan
-     assert(plan.isInstanceOf[AdaptiveSparkPlanExec])
-     val shuffle = plan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan.collect {
--      case s: ShuffleExchangeExec => s
-+      case s: ShuffleExchangeLike => s
-     }
-     assert(shuffle.size == 1)
-     assert(shuffle(0).outputPartitioning.numPartitions == numPartition)
-@@ -226,7 +240,8 @@ class AdaptiveQueryExecSuite
-       assert(smj.size == 1)
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
--      checkNumLocalShuffleReads(adaptivePlan)
-+      // Comet shuffle changes shuffle metrics
-+      // checkNumLocalShuffleReads(adaptivePlan)
-     }
-   }
- 
-@@ -253,7 +268,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Reuse the parallelism of coalesced shuffle in local shuffle read") {
-+  test("Reuse the parallelism of coalesced shuffle in local shuffle read",
-+      IgnoreComet("Comet shuffle changes shuffle partition size")) {
-     withSQLConf(
-       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-@@ -285,7 +301,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Reuse the default parallelism in local shuffle read") {
-+  test("Reuse the default parallelism in local shuffle read",
-+      IgnoreComet("Comet shuffle changes shuffle partition size")) {
-     withSQLConf(
-       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-@@ -299,7 +316,8 @@ class AdaptiveQueryExecSuite
-       val localReads = collect(adaptivePlan) {
-         case read: AQEShuffleReadExec if read.isLocalRead => read
-       }
--      assert(localReads.length == 2)
-+      // Comet shuffle changes shuffle metrics
-+      assert(localReads.length == 1)
-       val localShuffleRDD0 = localReads(0).execute().asInstanceOf[ShuffledRowRDD]
-       val localShuffleRDD1 = localReads(1).execute().asInstanceOf[ShuffledRowRDD]
-       // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2
-@@ -324,7 +342,9 @@ class AdaptiveQueryExecSuite
-           .groupBy($"a").count()
-         checkAnswer(testDf, Seq())
-         val plan = testDf.queryExecution.executedPlan
--        assert(find(plan)(_.isInstanceOf[SortMergeJoinExec]).isDefined)
-+        assert(find(plan) { case p =>
-+          p.isInstanceOf[SortMergeJoinExec] || p.isInstanceOf[CometSortMergeJoinExec]
-+        }.isDefined)
-         val coalescedReads = collect(plan) {
-           case r: AQEShuffleReadExec => r
-         }
-@@ -338,7 +358,9 @@ class AdaptiveQueryExecSuite
-           .groupBy($"a").count()
-         checkAnswer(testDf, Seq())
-         val plan = testDf.queryExecution.executedPlan
--        assert(find(plan)(_.isInstanceOf[BroadcastHashJoinExec]).isDefined)
-+        assert(find(plan) { case p =>
-+          p.isInstanceOf[BroadcastHashJoinExec] || p.isInstanceOf[CometBroadcastHashJoinExec]
-+        }.isDefined)
-         val coalescedReads = collect(plan) {
-           case r: AQEShuffleReadExec => r
-         }
-@@ -348,7 +370,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Scalar subquery") {
-+  test("Scalar subquery", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -363,7 +385,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Scalar subquery in later stages") {
-+  test("Scalar subquery in later stages", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -379,7 +401,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("multiple joins") {
-+  test("multiple joins", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -424,7 +446,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("multiple joins with aggregate") {
-+  test("multiple joins with aggregate", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -469,7 +491,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("multiple joins with aggregate 2") {
-+  test("multiple joins with aggregate 2", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") {
-@@ -515,7 +537,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Exchange reuse") {
-+  test("Exchange reuse", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -534,7 +556,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Exchange reuse with subqueries") {
-+  test("Exchange reuse with subqueries", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -565,7 +587,9 @@ class AdaptiveQueryExecSuite
-       assert(smj.size == 1)
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
--      checkNumLocalShuffleReads(adaptivePlan)
-+      // Comet shuffle changes shuffle metrics,
-+      // so we can't check the number of local shuffle reads.
-+      // checkNumLocalShuffleReads(adaptivePlan)
-       // Even with local shuffle read, the query stage reuse can also work.
-       val ex = findReusedExchange(adaptivePlan)
-       assert(ex.nonEmpty)
-@@ -586,7 +610,9 @@ class AdaptiveQueryExecSuite
-       assert(smj.size == 1)
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
--      checkNumLocalShuffleReads(adaptivePlan)
-+      // Comet shuffle changes shuffle metrics,
-+      // so we can't check the number of local shuffle reads.
-+      // checkNumLocalShuffleReads(adaptivePlan)
-       // Even with local shuffle read, the query stage reuse can also work.
-       val ex = findReusedExchange(adaptivePlan)
-       assert(ex.isEmpty)
-@@ -595,7 +621,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Broadcast exchange reuse across subqueries") {
-+  test("Broadcast exchange reuse across subqueries",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "20000000",
-@@ -690,7 +717,8 @@ class AdaptiveQueryExecSuite
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
-       // There is still a SMJ, and its two shuffles can't apply local read.
--      checkNumLocalShuffleReads(adaptivePlan, 2)
-+      // Comet shuffle changes shuffle metrics
-+      // checkNumLocalShuffleReads(adaptivePlan, 2)
-     }
-   }
- 
-@@ -812,7 +840,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-29544: adaptive skew join with different join types") {
-+  test("SPARK-29544: adaptive skew join with different join types",
-+      IgnoreComet("Comet shuffle has different partition metrics")) {
-     Seq("SHUFFLE_MERGE", "SHUFFLE_HASH").foreach { joinHint =>
-       def getJoinNode(plan: SparkPlan): Seq[ShuffledJoin] = if (joinHint == "SHUFFLE_MERGE") {
-         findTopLevelSortMergeJoin(plan)
-@@ -1030,7 +1059,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("metrics of the shuffle read") {
-+  test("metrics of the shuffle read",
-+      IgnoreComet("Comet shuffle changes the metrics")) {
-     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
-       val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
-         "SELECT key FROM testData GROUP BY key")
-@@ -1625,7 +1655,7 @@ class AdaptiveQueryExecSuite
-         val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
-           "SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id")
-         assert(collect(adaptivePlan) {
--          case s: ShuffleExchangeExec => s
-+          case s: ShuffleExchangeLike => s
-         }.length == 1)
-       }
-     }
-@@ -1705,7 +1735,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-33551: Do not use AQE shuffle read for repartition") {
-+  test("SPARK-33551: Do not use AQE shuffle read for repartition",
-+      IgnoreComet("Comet shuffle changes partition size")) {
-     def hasRepartitionShuffle(plan: SparkPlan): Boolean = {
-       find(plan) {
-         case s: ShuffleExchangeLike =>
-@@ -1890,6 +1921,9 @@ class AdaptiveQueryExecSuite
-     def checkNoCoalescePartitions(ds: Dataset[Row], origin: ShuffleOrigin): Unit = {
-       assert(collect(ds.queryExecution.executedPlan) {
-         case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s
-+        case c: CometShuffleExchangeExec
-+          if c.originalPlan.shuffleOrigin == origin &&
-+            c.originalPlan.numPartitions == 2 => c
-       }.size == 1)
-       ds.collect()
-       val plan = ds.queryExecution.executedPlan
-@@ -1898,6 +1932,9 @@ class AdaptiveQueryExecSuite
-       }.isEmpty)
-       assert(collect(plan) {
-         case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s
-+        case c: CometShuffleExchangeExec
-+          if c.originalPlan.shuffleOrigin == origin &&
-+            c.originalPlan.numPartitions == 2 => c
-       }.size == 1)
-       checkAnswer(ds, testData)
-     }
-@@ -2054,7 +2091,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-35264: Support AQE side shuffled hash join formula") {
-+  test("SPARK-35264: Support AQE side shuffled hash join formula",
-+      IgnoreComet("Comet shuffle changes the partition size")) {
-     withTempView("t1", "t2") {
-       def checkJoinStrategy(shouldShuffleHashJoin: Boolean): Unit = {
-         Seq("100", "100000").foreach { size =>
-@@ -2140,7 +2178,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-35725: Support optimize skewed partitions in RebalancePartitions") {
-+  test("SPARK-35725: Support optimize skewed partitions in RebalancePartitions",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withTempView("v") {
-       withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-@@ -2239,7 +2278,7 @@ class AdaptiveQueryExecSuite
-               runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM skewData1 " +
-                 s"JOIN skewData2 ON key1 = key2 GROUP BY key1")
-             val shuffles1 = collect(adaptive1) {
--              case s: ShuffleExchangeExec => s
-+              case s: ShuffleExchangeLike => s
-             }
-             assert(shuffles1.size == 3)
-             // shuffles1.head is the top-level shuffle under the Aggregate operator
-@@ -2252,7 +2291,7 @@ class AdaptiveQueryExecSuite
-               runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM skewData1 " +
-                 s"JOIN skewData2 ON key1 = key2")
-             val shuffles2 = collect(adaptive2) {
--              case s: ShuffleExchangeExec => s
-+              case s: ShuffleExchangeLike => s
-             }
-             if (hasRequiredDistribution) {
-               assert(shuffles2.size == 3)
-@@ -2286,7 +2325,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-35794: Allow custom plugin for cost evaluator") {
-+  test("SPARK-35794: Allow custom plugin for cost evaluator",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     CostEvaluator.instantiate(
-       classOf[SimpleShuffleSortCostEvaluator].getCanonicalName, spark.sparkContext.getConf)
-     intercept[IllegalArgumentException] {
-@@ -2417,7 +2457,8 @@ class AdaptiveQueryExecSuite
-   }
- 
-   test("SPARK-48037: Fix SortShuffleWriter lacks shuffle write related metrics " +
--    "resulting in potentially inaccurate data") {
-+    "resulting in potentially inaccurate data",
-+    IgnoreComet("https://github.com/apache/datafusion-comet/issues/1501")) {
-     withTable("t3") {
-       withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-@@ -2452,6 +2493,7 @@ class AdaptiveQueryExecSuite
-           val (_, adaptive) = runAdaptiveAndVerifyResult(query)
-           assert(adaptive.collect {
-             case sort: SortExec => sort
-+            case sort: CometSortExec => sort
-           }.size == 1)
-           val read = collect(adaptive) {
-             case read: AQEShuffleReadExec => read
-@@ -2469,7 +2511,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-37357: Add small partition factor for rebalance partitions") {
-+  test("SPARK-37357: Add small partition factor for rebalance partitions",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withTempView("v") {
-       withSQLConf(
-         SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED.key -> "true",
-@@ -2581,7 +2624,7 @@ class AdaptiveQueryExecSuite
-           runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
-             "JOIN skewData3 ON value2 = value3")
-         val shuffles1 = collect(adaptive1) {
--          case s: ShuffleExchangeExec => s
-+          case s: ShuffleExchangeLike => s
-         }
-         assert(shuffles1.size == 4)
-         val smj1 = findTopLevelSortMergeJoin(adaptive1)
-@@ -2592,7 +2635,7 @@ class AdaptiveQueryExecSuite
-           runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
-             "JOIN skewData3 ON value1 = value3")
-         val shuffles2 = collect(adaptive2) {
--          case s: ShuffleExchangeExec => s
-+          case s: ShuffleExchangeLike => s
-         }
-         assert(shuffles2.size == 4)
-         val smj2 = findTopLevelSortMergeJoin(adaptive2)
-@@ -2850,6 +2893,7 @@ class AdaptiveQueryExecSuite
-         }.size == (if (firstAccess) 1 else 0))
-         assert(collect(initialExecutedPlan) {
-           case s: SortExec => s
-+          case s: CometSortExec => s
-         }.size == (if (firstAccess) 2 else 0))
-         assert(collect(initialExecutedPlan) {
-           case i: InMemoryTableScanLike => i
-@@ -2980,7 +3024,9 @@ class AdaptiveQueryExecSuite
- 
-       val plan = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec]
-       assert(plan.inputPlan.isInstanceOf[TakeOrderedAndProjectExec])
--      assert(plan.finalPhysicalPlan.isInstanceOf[WindowExec])
-+      assert(
-+        plan.finalPhysicalPlan.isInstanceOf[WindowExec] ||
-+        plan.finalPhysicalPlan.find(_.isInstanceOf[CometWindowExec]).nonEmpty)
-       plan.inputPlan.output.zip(plan.finalPhysicalPlan.output).foreach { case (o1, o2) =>
-         assert(o1.semanticEquals(o2), "Different output column order after AQE optimization")
-       }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-index 05872d41131..a2c328b9742 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-@@ -21,7 +21,7 @@ import java.io.File
- 
- import org.apache.hadoop.fs.{FileStatus, Path}
- 
--import org.apache.spark.sql.{DataFrame, Dataset, QueryTest, Row}
-+import org.apache.spark.sql.{DataFrame, Dataset, IgnoreComet, QueryTest, Row}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{Expression, FileSourceConstantMetadataStructField, FileSourceGeneratedMetadataStructField, Literal}
- import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
-@@ -134,7 +134,8 @@ class FileSourceCustomMetadataStructSuite extends QueryTest with SharedSparkSess
-     }
-   }
- 
--  test("[SPARK-43226] extra constant metadata fields with extractors") {
-+  test("[SPARK-43226] extra constant metadata fields with extractors",
-+      IgnoreComet("TODO: fix Comet for this test")) {
-     withTempData("parquet", FILE_SCHEMA) { (_, f0, f1) =>
-       val format = new TestFileFormat(extraConstantMetadataFields) {
-         val extractPartitionNumber = { pf: PartitionedFile =>
-@@ -335,7 +336,8 @@ class FileSourceCustomMetadataStructSuite extends QueryTest with SharedSparkSess
-     }
-   }
- 
--  test("generated columns and extractors take precedence over metadata map values") {
-+  test("generated columns and extractors take precedence over metadata map values",
-+      IgnoreComet("TODO: fix Comet for this test")) {
-     withTempData("parquet", FILE_SCHEMA) { (_, f0, f1) =>
-       import FileFormat.{FILE_NAME, FILE_SIZE}
-       import ParquetFileFormat.ROW_INDEX
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
-index bf496d6db21..9bb57a9b4c6 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
-@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.Concat
- import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
- import org.apache.spark.sql.catalyst.plans.logical.Expand
- import org.apache.spark.sql.catalyst.types.DataTypeUtils
-+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec}
- import org.apache.spark.sql.execution.FileSourceScanExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.functions._
-@@ -868,6 +869,8 @@ abstract class SchemaPruningSuite
-     val fileSourceScanSchemata =
-       collect(df.queryExecution.executedPlan) {
-         case scan: FileSourceScanExec => scan.requiredSchema
-+        case scan: CometScanExec => scan.requiredSchema
-+        case scan: CometNativeScanExec => scan.requiredSchema
-       }
-     assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
-       s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala
-index ce43edb79c1..8436cb727c6 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala
-@@ -17,9 +17,10 @@
- 
- package org.apache.spark.sql.execution.datasources
- 
--import org.apache.spark.sql.{QueryTest, Row}
-+import org.apache.spark.sql.{IgnoreComet, QueryTest, Row}
- import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, NullsFirst, SortOrder}
- import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Sort}
-+import org.apache.spark.sql.comet.CometSortExec
- import org.apache.spark.sql.execution.{QueryExecution, SortExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
- import org.apache.spark.sql.internal.SQLConf
-@@ -225,6 +226,7 @@ class V1WriteCommandSuite extends QueryTest with SharedSparkSession with V1Write
-           // assert the outer most sort in the executed plan
-           assert(plan.collectFirst {
-             case s: SortExec => s
-+            case s: CometSortExec => s.originalPlan.asInstanceOf[SortExec]
-           }.exists {
-             case SortExec(Seq(
-               SortOrder(AttributeReference("key", IntegerType, _, _), Ascending, NullsFirst, _),
-@@ -272,6 +274,7 @@ class V1WriteCommandSuite extends QueryTest with SharedSparkSession with V1Write
-         // assert the outer most sort in the executed plan
-         assert(plan.collectFirst {
-           case s: SortExec => s
-+          case s: CometSortExec => s.originalPlan.asInstanceOf[SortExec]
-         }.exists {
-           case SortExec(Seq(
-             SortOrder(AttributeReference("value", StringType, _, _), Ascending, NullsFirst, _),
-@@ -305,7 +308,8 @@ class V1WriteCommandSuite extends QueryTest with SharedSparkSession with V1Write
-     }
-   }
- 
--  test("v1 write with AQE changing SMJ to BHJ") {
-+  test("v1 write with AQE changing SMJ to BHJ",
-+      IgnoreComet("TODO: Comet SMJ to BHJ by AQE")) {
-     withPlannedWrite { enabled =>
-       withTable("t") {
-         sql(
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
-index 0b6fdef4f74..5b18c55da4b 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
-@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, GlobFilter, Path}
- import org.mockito.Mockito.{mock, when}
- 
- import org.apache.spark.SparkException
--import org.apache.spark.sql.{DataFrame, QueryTest, Row}
-+import org.apache.spark.sql.{DataFrame, IgnoreCometSuite, QueryTest, Row}
- import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
- import org.apache.spark.sql.execution.datasources.PartitionedFile
- import org.apache.spark.sql.functions.col
-@@ -38,7 +38,9 @@ import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.sql.types._
- import org.apache.spark.util.Utils
- 
--class BinaryFileFormatSuite extends QueryTest with SharedSparkSession {
-+// For some reason this suite is flaky w/ or w/o Comet when running in Github workflow.
-+// Since it isn't related to Comet, we disable it for now.
-+class BinaryFileFormatSuite extends QueryTest with SharedSparkSession with IgnoreCometSuite {
-   import BinaryFileFormat._
- 
-   private var testDir: String = _
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
-index 07e2849ce6f..3e73645b638 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
-@@ -28,7 +28,7 @@ import org.apache.parquet.hadoop.ParquetOutputFormat
- 
- import org.apache.spark.TestUtils
- import org.apache.spark.memory.MemoryMode
--import org.apache.spark.sql.Row
-+import org.apache.spark.sql.{IgnoreComet, Row}
- import org.apache.spark.sql.catalyst.util.DateTimeUtils
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
-@@ -201,7 +201,8 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
-     }
-   }
- 
--  test("parquet v2 pages - rle encoding for boolean value columns") {
-+  test("parquet v2 pages - rle encoding for boolean value columns",
-+      IgnoreComet("Comet doesn't support RLE encoding yet")) {
-     val extraOptions = Map[String, String](
-       ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
-     )
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 8e88049f51e..686e2e9c9b0 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-@@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-           // When a filter is pushed to Parquet, Parquet can apply it to every row.
-           // So, we can check the number of rows returned from the Parquet
-           // to make sure our filter pushdown work.
--          assert(stripSparkFilter(df).count == 1)
-+          // Similar to Spark's vectorized reader, Comet doesn't do row-level filtering but relies
-+          // on Spark to apply the data filters after columnar batches are returned
-+          if (!isCometEnabled) {
-+            assert(stripSparkFilter(df).count == 1)
-+          }
-         }
-       }
-     }
-@@ -1498,7 +1502,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("Filters should be pushed down for vectorized Parquet reader at row group level") {
-+  test("Filters should be pushed down for vectorized Parquet reader at row group level",
-+    IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
-     import testImplicits._
- 
-     withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
-@@ -1580,7 +1585,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-           // than the total length but should not be a single record.
-           // Note that, if record level filtering is enabled, it should be a single record.
-           // If no filter is pushed down to Parquet, it should be the total length of data.
--          assert(actual > 1 && actual < data.length)
-+          // Only enable Comet test iff it's scan only, since with native execution
-+          // `stripSparkFilter` can't remove the native filter
-+          if (!isCometEnabled || isCometScanOnly) {
-+            assert(actual > 1 && actual < data.length)
-+          }
-         }
-       }
-     }
-@@ -1607,7 +1616,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-         // than the total length but should not be a single record.
-         // Note that, if record level filtering is enabled, it should be a single record.
-         // If no filter is pushed down to Parquet, it should be the total length of data.
--        assert(actual > 1 && actual < data.length)
-+        // Only enable Comet test iff it's scan only, since with native execution
-+        // `stripSparkFilter` can't remove the native filter
-+        if (!isCometEnabled || isCometScanOnly) {
-+          assert(actual > 1 && actual < data.length)
-+        }
-       }
-     }
-   }
-@@ -1743,7 +1756,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
-+  test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+      IgnoreComet("IN predicate is not yet supported in Comet, see issue #36")) {
-     val schema = StructType(Seq(
-       StructField("a", IntegerType, nullable = false)
-     ))
-@@ -1984,7 +1998,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("Support Parquet column index") {
-+  test("Support Parquet column index",
-+      IgnoreComet("Comet doesn't support Parquet column index yet")) {
-     // block 1:
-     //                      null count  min                                       max
-     // page-0                         0  0                                         99
-@@ -2044,7 +2059,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("SPARK-34562: Bloom filter push down") {
-+  test("SPARK-34562: Bloom filter push down",
-+    IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
-     withTempPath { dir =>
-       val path = dir.getCanonicalPath
-       spark.range(100).selectExpr("id * 2 AS id")
-@@ -2276,7 +2292,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
-           assert(pushedParquetFilters.exists(_.getClass === filterClass),
-             s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
- 
--          checker(stripSparkFilter(query), expected)
-+          // Similar to Spark's vectorized reader, Comet doesn't do row-level filtering but relies
-+          // on Spark to apply the data filters after columnar batches are returned
-+          if (!isCometEnabled) {
-+            checker(stripSparkFilter(query), expected)
-+          }
-         } else {
-           assert(selectedFilters.isEmpty, "There is filter pushed down")
-         }
-@@ -2336,7 +2356,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
-           assert(pushedParquetFilters.exists(_.getClass === filterClass),
-             s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
- 
--          checker(stripSparkFilter(query), expected)
-+          // Similar to Spark's vectorized reader, Comet doesn't do row-level filtering but relies
-+          // on Spark to apply the data filters after columnar batches are returned
-+          if (!isCometEnabled) {
-+            checker(stripSparkFilter(query), expected)
-+          }
- 
-         case _ =>
-           throw new AnalysisException("Can not match ParquetTable in the query.")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-index 4f8a9e39716..fb55ac7a955 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-@@ -1335,7 +1335,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
-     }
-   }
- 
--  test("SPARK-40128 read DELTA_LENGTH_BYTE_ARRAY encoded strings") {
-+  test("SPARK-40128 read DELTA_LENGTH_BYTE_ARRAY encoded strings",
-+      IgnoreComet("Comet doesn't support DELTA encoding yet")) {
-     withAllParquetReaders {
-       checkAnswer(
-         // "fruit" column in this file is encoded using DELTA_LENGTH_BYTE_ARRAY.
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index f6472ba3d9d..7a8f5317ed7 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-@@ -998,7 +998,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-     }
-   }
- 
--  test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") {
-+  test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups",
-+    IgnoreCometNativeScan("Native scans had the filter pushed into DF operator, cannot strip")) {
-     withAllParquetReaders {
-       withTempPath { path =>
-         // Repeated values for dictionary encoding.
-@@ -1067,7 +1068,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-         checkAnswer(readParquet(schema, path), df)
-       }
- 
--      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
-+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
-+          "spark.comet.enabled" -> "false") {
-         val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
-         checkAnswer(readParquet(schema1, path), df)
-         val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
-@@ -1089,7 +1091,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-       val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c, CAST('1.2' AS BINARY) d")
-       df.write.parquet(path.toString)
- 
--      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
-+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
-+          "spark.comet.enabled" -> "false") {
-         checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00"))
-         checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null))
-         checkAnswer(readParquet("b DECIMAL(11, 1)", path), sql("SELECT 123456.0"))
-@@ -1148,7 +1151,7 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-             .where(s"a < ${Long.MaxValue}")
-             .collect()
-         }
--        assert(exception.getCause.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
-+        assert(exception.getMessage.contains("Column: [a], Expected: bigint, Found: INT32"))
-       }
-     }
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
-index 4f906411345..6cc69f7e915 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
-@@ -21,7 +21,7 @@ import java.nio.file.{Files, Paths, StandardCopyOption}
- import java.sql.{Date, Timestamp}
- 
- import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException, SparkUpgradeException}
--import org.apache.spark.sql.{QueryTest, Row, SPARK_LEGACY_DATETIME_METADATA_KEY, SPARK_LEGACY_INT96_METADATA_KEY, SPARK_TIMEZONE_METADATA_KEY}
-+import org.apache.spark.sql.{IgnoreCometSuite, QueryTest, Row, SPARK_LEGACY_DATETIME_METADATA_KEY, SPARK_LEGACY_INT96_METADATA_KEY, SPARK_TIMEZONE_METADATA_KEY}
- import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
- import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}
- import org.apache.spark.sql.internal.LegacyBehaviorPolicy.{CORRECTED, EXCEPTION, LEGACY}
-@@ -30,9 +30,11 @@ import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType.{INT96,
- import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.tags.SlowSQLTest
- 
-+// Comet is disabled for this suite because it doesn't support datetime rebase mode
- abstract class ParquetRebaseDatetimeSuite
-   extends QueryTest
-   with ParquetTest
-+  with IgnoreCometSuite
-   with SharedSparkSession {
- 
-   import testImplicits._
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-index 27c2a2148fd..df04a15fb1f 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-@@ -20,12 +20,14 @@ import java.io.File
- 
- import scala.collection.JavaConverters._
- 
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.apache.parquet.column.ParquetProperties._
- import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
- import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
- 
- import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometScanExec}
- import org.apache.spark.sql.execution.FileSourceScanExec
- import org.apache.spark.sql.execution.datasources.FileFormat
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -172,6 +174,8 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
- 
-   private def testRowIndexGeneration(label: String, conf: RowIndexTestConf): Unit = {
-     test (s"$label - ${conf.desc}") {
-+      // native_datafusion Parquet scan does not support row index generation.
-+      assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_DATAFUSION)
-       withSQLConf(conf.sqlConfs: _*) {
-         withTempPath { path =>
-           // Read row index using _metadata.row_index if that is supported by the file format.
-@@ -243,6 +247,12 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
-             case f: FileSourceScanExec =>
-               numPartitions += f.inputRDD.partitions.length
-               numOutputRows += f.metrics("numOutputRows").value
-+            case b: CometScanExec =>
-+              numPartitions += b.inputRDD.partitions.length
-+              numOutputRows += b.metrics("numOutputRows").value
-+            case b: CometBatchScanExec =>
-+              numPartitions += b.inputRDD.partitions.length
-+              numOutputRows += b.metrics("numOutputRows").value
-             case _ =>
-           }
-           assert(numPartitions > 0)
-@@ -301,6 +311,8 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
-     val conf = RowIndexTestConf(useDataSourceV2 = useDataSourceV2)
- 
-     test(s"invalid row index column type - ${conf.desc}") {
-+      // native_datafusion Parquet scan does not support row index generation.
-+      assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_DATAFUSION)
-       withSQLConf(conf.sqlConfs: _*) {
-         withTempPath{ path =>
-           val df = spark.range(0, 10, 1, 1).toDF("id")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
-index 5c0b7def039..151184bc98c 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
-@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
- import org.apache.spark.SparkConf
- import org.apache.spark.sql.DataFrame
- import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-+import org.apache.spark.sql.comet.CometBatchScanExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.datasources.SchemaPruningSuite
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -56,6 +57,7 @@ class ParquetV2SchemaPruningSuite extends ParquetSchemaPruningSuite {
-     val fileSourceScanSchemata =
-       collect(df.queryExecution.executedPlan) {
-         case scan: BatchScanExec => scan.scan.asInstanceOf[ParquetScan].readDataSchema
-+        case scan: CometBatchScanExec => scan.scan.asInstanceOf[ParquetScan].readDataSchema
-       }
-     assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
-       s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-index 3f47c5e506f..bc1ee1ec0ba 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-@@ -27,6 +27,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
- import org.apache.parquet.schema.Type._
- 
- import org.apache.spark.SparkException
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.expressions.Cast.toSQLType
- import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
- import org.apache.spark.sql.functions.desc
-@@ -1036,7 +1037,8 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
-     e
-   }
- 
--  test("schema mismatch failure error message for parquet reader") {
-+  test("schema mismatch failure error message for parquet reader",
-+      IgnoreComet("Comet doesn't work with vectorizedReaderEnabled = false")) {
-     withTempPath { dir =>
-       val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = false)
-       val expectedMessage = "Encountered error while reading file"
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
-index b8f3ea3c6f3..bbd44221288 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
-@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.debug
- import java.io.ByteArrayOutputStream
- 
- import org.apache.spark.rdd.RDD
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.Attribute
- import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
-@@ -125,7 +126,8 @@ class DebuggingSuite extends DebuggingSuiteBase with DisableAdaptiveExecutionSui
-          | id LongType: {}""".stripMargin))
-   }
- 
--  test("SPARK-28537: DebugExec cannot debug columnar related queries") {
-+  test("SPARK-28537: DebugExec cannot debug columnar related queries",
-+      IgnoreComet("Comet does not use FileScan")) {
-     withTempPath { workDir =>
-       val workDirPath = workDir.getAbsolutePath
-       val input = spark.range(5).toDF("id")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
-index 5cdbdc27b32..307fba16578 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
-@@ -46,8 +46,10 @@ import org.apache.spark.sql.util.QueryExecutionListener
- import org.apache.spark.util.{AccumulatorContext, JsonProtocol}
- 
- // Disable AQE because metric info is different with AQE on/off
-+// This test suite runs tests against the metrics of physical operators.
-+// Disabling it for Comet because the metrics are different with Comet enabled.
- class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
--  with DisableAdaptiveExecutionSuite {
-+  with DisableAdaptiveExecutionSuite with IgnoreCometSuite {
-   import testImplicits._
- 
-   /**
-@@ -765,7 +767,8 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
-     }
-   }
- 
--  test("SPARK-26327: FileSourceScanExec metrics") {
-+  test("SPARK-26327: FileSourceScanExec metrics",
-+      IgnoreComet("Spark uses row-based Parquet reader while Comet is vectorized")) {
-     withTable("testDataForScan") {
-       spark.range(10).selectExpr("id", "id % 3 as p")
-         .write.partitionBy("p").saveAsTable("testDataForScan")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
-index 0ab8691801d..d9125f658ad 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.execution.python
- 
- import org.apache.spark.sql.catalyst.plans.logical.{ArrowEvalPython, BatchEvalPython, Limit, LocalLimit}
-+import org.apache.spark.sql.comet._
- import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan, SparkPlanTest}
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
-@@ -108,6 +109,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: FileSourceScanExec => scan
-+            case scan: CometScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           assert(scanNodes.head.output.map(_.name) == Seq("a"))
-@@ -120,11 +122,16 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: FileSourceScanExec => scan
-+            case scan: CometScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           // $"a" is not null and $"a" > 1
--          assert(scanNodes.head.dataFilters.length == 2)
--          assert(scanNodes.head.dataFilters.flatMap(_.references.map(_.name)).distinct == Seq("a"))
-+          val dataFilters = scanNodes.head match {
-+            case scan: FileSourceScanExec => scan.dataFilters
-+            case scan: CometScanExec => scan.dataFilters
-+          }
-+          assert(dataFilters.length == 2)
-+          assert(dataFilters.flatMap(_.references.map(_.name)).distinct == Seq("a"))
-         }
-       }
-     }
-@@ -145,6 +152,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: BatchScanExec => scan
-+            case scan: CometBatchScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           assert(scanNodes.head.output.map(_.name) == Seq("a"))
-@@ -157,6 +165,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: BatchScanExec => scan
-+            case scan: CometBatchScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           // $"a" is not null and $"a" > 1
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala
-index d083cac48ff..3c11bcde807 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala
-@@ -37,8 +37,10 @@ import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryException,
- import org.apache.spark.sql.streaming.util.StreamManualClock
- import org.apache.spark.util.Utils
- 
-+// For some reason this suite is flaky w/ or w/o Comet when running in Github workflow.
-+// Since it isn't related to Comet, we disable it for now.
- class AsyncProgressTrackingMicroBatchExecutionSuite
--  extends StreamTest with BeforeAndAfter with Matchers {
-+  extends StreamTest with BeforeAndAfter with Matchers with IgnoreCometSuite {
- 
-   import testImplicits._
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
-index 746f289c393..0c99d028163 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
-@@ -25,10 +25,11 @@ import org.apache.spark.sql.catalyst.expressions
- import org.apache.spark.sql.catalyst.expressions._
- import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
- import org.apache.spark.sql.catalyst.types.DataTypeUtils
--import org.apache.spark.sql.execution.{FileSourceScanExec, SortExec, SparkPlan}
-+import org.apache.spark.sql.comet._
-+import org.apache.spark.sql.execution.{ColumnarToRowExec, FileSourceScanExec, SortExec, SparkPlan}
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper}
- import org.apache.spark.sql.execution.datasources.BucketingUtils
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.joins.SortMergeJoinExec
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -102,12 +103,20 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-     }
-   }
- 
--  private def getFileScan(plan: SparkPlan): FileSourceScanExec = {
--    val fileScan = collect(plan) { case f: FileSourceScanExec => f }
-+  private def getFileScan(plan: SparkPlan): SparkPlan = {
-+    val fileScan = collect(plan) {
-+      case f: FileSourceScanExec => f
-+      case f: CometScanExec => f
-+    }
-     assert(fileScan.nonEmpty, plan)
-     fileScan.head
-   }
- 
-+  private def getBucketScan(plan: SparkPlan): Boolean = getFileScan(plan) match {
-+    case fs: FileSourceScanExec => fs.bucketedScan
-+    case bs: CometScanExec => bs.bucketedScan
-+  }
-+
-   // To verify if the bucket pruning works, this function checks two conditions:
-   //   1) Check if the pruned buckets (before filtering) are empty.
-   //   2) Verify the final result is the same as the expected one
-@@ -156,7 +165,8 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-           val planWithoutBucketedScan = bucketedDataFrame.filter(filterCondition)
-             .queryExecution.executedPlan
-           val fileScan = getFileScan(planWithoutBucketedScan)
--          assert(!fileScan.bucketedScan, s"except no bucketed scan but found\n$fileScan")
-+          val bucketedScan = getBucketScan(planWithoutBucketedScan)
-+          assert(!bucketedScan, s"except no bucketed scan but found\n$fileScan")
- 
-           val bucketColumnType = bucketedDataFrame.schema.apply(bucketColumnIndex).dataType
-           val rowsWithInvalidBuckets = fileScan.execute().filter(row => {
-@@ -452,28 +462,49 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-         val joinOperator = if (joined.sqlContext.conf.adaptiveExecutionEnabled) {
-           val executedPlan =
-             joined.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
--          assert(executedPlan.isInstanceOf[SortMergeJoinExec])
--          executedPlan.asInstanceOf[SortMergeJoinExec]
-+          executedPlan match {
-+            case s: SortMergeJoinExec => s
-+            case b: CometSortMergeJoinExec =>
-+              b.originalPlan match {
-+                case s: SortMergeJoinExec => s
-+                case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+              }
-+            case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+          }
-         } else {
-           val executedPlan = joined.queryExecution.executedPlan
--          assert(executedPlan.isInstanceOf[SortMergeJoinExec])
--          executedPlan.asInstanceOf[SortMergeJoinExec]
-+          executedPlan match {
-+            case s: SortMergeJoinExec => s
-+            case ColumnarToRowExec(child) =>
-+              child.asInstanceOf[CometSortMergeJoinExec].originalPlan match {
-+                case s: SortMergeJoinExec => s
-+                case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+              }
-+            case CometColumnarToRowExec(child) =>
-+              child.asInstanceOf[CometSortMergeJoinExec].originalPlan match {
-+                case s: SortMergeJoinExec => s
-+                case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+              }
-+            case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+          }
-         }
- 
-         // check existence of shuffle
-         assert(
--          joinOperator.left.exists(_.isInstanceOf[ShuffleExchangeExec]) == shuffleLeft,
-+          joinOperator.left.exists(op => op.isInstanceOf[ShuffleExchangeLike]) == shuffleLeft,
-           s"expected shuffle in plan to be $shuffleLeft but found\n${joinOperator.left}")
-         assert(
--          joinOperator.right.exists(_.isInstanceOf[ShuffleExchangeExec]) == shuffleRight,
-+          joinOperator.right.exists(op => op.isInstanceOf[ShuffleExchangeLike]) == shuffleRight,
-           s"expected shuffle in plan to be $shuffleRight but found\n${joinOperator.right}")
- 
-         // check existence of sort
-         assert(
--          joinOperator.left.exists(_.isInstanceOf[SortExec]) == sortLeft,
-+          joinOperator.left.exists(op => op.isInstanceOf[SortExec] || op.isInstanceOf[CometExec] &&
-+            op.asInstanceOf[CometExec].originalPlan.isInstanceOf[SortExec]) == sortLeft,
-           s"expected sort in the left child to be $sortLeft but found\n${joinOperator.left}")
-         assert(
--          joinOperator.right.exists(_.isInstanceOf[SortExec]) == sortRight,
-+          joinOperator.right.exists(op => op.isInstanceOf[SortExec] || op.isInstanceOf[CometExec] &&
-+            op.asInstanceOf[CometExec].originalPlan.isInstanceOf[SortExec]) == sortRight,
-           s"expected sort in the right child to be $sortRight but found\n${joinOperator.right}")
- 
-         // check the output partitioning
-@@ -836,11 +867,11 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-       df1.write.format("parquet").bucketBy(8, "i").saveAsTable("bucketed_table")
- 
-       val scanDF = spark.table("bucketed_table").select("j")
--      assert(!getFileScan(scanDF.queryExecution.executedPlan).bucketedScan)
-+      assert(!getBucketScan(scanDF.queryExecution.executedPlan))
-       checkAnswer(scanDF, df1.select("j"))
- 
-       val aggDF = spark.table("bucketed_table").groupBy("j").agg(max("k"))
--      assert(!getFileScan(aggDF.queryExecution.executedPlan).bucketedScan)
-+      assert(!getBucketScan(aggDF.queryExecution.executedPlan))
-       checkAnswer(aggDF, df1.groupBy("j").agg(max("k")))
-     }
-   }
-@@ -1029,15 +1060,21 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-           Seq(true, false).foreach { aqeEnabled =>
-             withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> aqeEnabled.toString) {
-               val plan = sql(query).queryExecution.executedPlan
--              val shuffles = collect(plan) { case s: ShuffleExchangeExec => s }
-+              val shuffles = collect(plan) { case s: ShuffleExchangeLike => s }
-               assert(shuffles.length == expectedNumShuffles)
- 
-               val scans = collect(plan) {
-                 case f: FileSourceScanExec if f.optionalNumCoalescedBuckets.isDefined => f
-+                case b: CometScanExec if b.optionalNumCoalescedBuckets.isDefined => b
-               }
-               if (expectedCoalescedNumBuckets.isDefined) {
-                 assert(scans.length == 1)
--                assert(scans.head.optionalNumCoalescedBuckets == expectedCoalescedNumBuckets)
-+                scans.head match {
-+                  case f: FileSourceScanExec =>
-+                    assert(f.optionalNumCoalescedBuckets == expectedCoalescedNumBuckets)
-+                  case b: CometScanExec =>
-+                    assert(b.optionalNumCoalescedBuckets == expectedCoalescedNumBuckets)
-+                }
-               } else {
-                 assert(scans.isEmpty)
-               }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
-index 6f897a9c0b7..b0723634f68 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
-@@ -20,6 +20,7 @@ package org.apache.spark.sql.sources
- import java.io.File
- 
- import org.apache.spark.SparkException
-+import org.apache.spark.sql.IgnoreCometSuite
- import org.apache.spark.sql.catalyst.TableIdentifier
- import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTableType}
- import org.apache.spark.sql.catalyst.parser.ParseException
-@@ -27,7 +28,10 @@ import org.apache.spark.sql.internal.SQLConf.BUCKETING_MAX_BUCKETS
- import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.util.Utils
- 
--class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession {
-+// For some reason this suite is flaky w/ or w/o Comet when running in Github workflow.
-+// Since it isn't related to Comet, we disable it for now.
-+class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession
-+    with IgnoreCometSuite {
-   import testImplicits._
- 
-   protected override lazy val sql = spark.sql _
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
-index d675503a8ba..659fa686fb7 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.sources
- 
- import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.execution.FileSourceScanExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.internal.SQLConf
-@@ -68,7 +69,10 @@ abstract class DisableUnnecessaryBucketedScanSuite
- 
-     def checkNumBucketedScan(query: String, expectedNumBucketedScan: Int): Unit = {
-       val plan = sql(query).queryExecution.executedPlan
--      val bucketedScan = collect(plan) { case s: FileSourceScanExec if s.bucketedScan => s }
-+      val bucketedScan = collect(plan) {
-+        case s: FileSourceScanExec if s.bucketedScan => s
-+        case s: CometScanExec if s.bucketedScan => s
-+      }
-       assert(bucketedScan.length == expectedNumBucketedScan)
-     }
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
-index 1954cce7fdc..73d1464780e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
-@@ -34,6 +34,7 @@ import org.apache.spark.paths.SparkPath
- import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
- import org.apache.spark.sql.{AnalysisException, DataFrame}
- import org.apache.spark.sql.catalyst.util.stringToFile
-+import org.apache.spark.sql.comet.CometBatchScanExec
- import org.apache.spark.sql.execution.DataSourceScanExec
- import org.apache.spark.sql.execution.datasources._
- import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV2Relation, FileScan, FileTable}
-@@ -761,6 +762,8 @@ class FileStreamSinkV2Suite extends FileStreamSinkSuite {
-       val fileScan = df.queryExecution.executedPlan.collect {
-         case batch: BatchScanExec if batch.scan.isInstanceOf[FileScan] =>
-           batch.scan.asInstanceOf[FileScan]
-+        case batch: CometBatchScanExec if batch.scan.isInstanceOf[FileScan] =>
-+          batch.scan.asInstanceOf[FileScan]
-       }.headOption.getOrElse {
-         fail(s"No FileScan in query\n${df.queryExecution}")
-       }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-index b597a244710..b2e8be41065 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-@@ -21,6 +21,7 @@ import java.io.File
- 
- import org.apache.commons.io.FileUtils
- 
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.{FlatMapGroupsWithStateExec, MemoryStream}
- import org.apache.spark.sql.internal.SQLConf
-@@ -91,7 +92,7 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require StatefulOpClusteredDistribution " +
--    "from children - without initial state") {
-+    "from children - without initial state", IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -243,7 +244,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in 3.2.x - without initial state") {
-+    "from children if the query starts from checkpoint in 3.2.x - without initial state",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -335,7 +337,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in prior to 3.2") {
-+    "from children if the query starts from checkpoint in prior to 3.2",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-index a3774bf17e6..6879c71037d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-@@ -25,7 +25,7 @@ import org.scalatest.exceptions.TestFailedException
- 
- import org.apache.spark.SparkException
- import org.apache.spark.api.java.function.FlatMapGroupsWithStateFunction
--import org.apache.spark.sql.{DataFrame, Encoder}
-+import org.apache.spark.sql.{DataFrame, Encoder, IgnoreCometSuite}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow}
- import org.apache.spark.sql.catalyst.plans.logical.FlatMapGroupsWithState
-@@ -46,8 +46,9 @@ case class RunningCount(count: Long)
- 
- case class Result(key: Long, count: Int)
- 
-+// TODO: fix Comet to enable this suite
- @SlowSQLTest
--class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
-+class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest with IgnoreCometSuite {
- 
-   import testImplicits._
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-index 2a2a83d35e1..e3b7b290b3e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.streaming
- 
- import org.apache.spark.SparkException
--import org.apache.spark.sql.{AnalysisException, Dataset, KeyValueGroupedDataset}
-+import org.apache.spark.sql.{AnalysisException, Dataset, IgnoreComet, KeyValueGroupedDataset}
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.MemoryStream
- import org.apache.spark.sql.execution.streaming.state.FlatMapGroupsWithStateExecHelper
-@@ -253,7 +253,8 @@ class FlatMapGroupsWithStateWithInitialStateSuite extends StateStoreMetricsTest
-     assert(e.message.contains(expectedError))
-   }
- 
--  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState") {
-+  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     val initialStateDS = Seq(("keyInStateAndData", new RunningCount(1))).toDS()
-     val initialState: KeyValueGroupedDataset[String, RunningCount] =
-       initialStateDS.groupByKey(_._1).mapValues(_._2)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
-index c97979a57a5..45a998db0e0 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
-@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Range, RepartitionByExpressi
- import org.apache.spark.sql.catalyst.streaming.{InternalOutputModes, StreamingRelationV2}
- import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes
- import org.apache.spark.sql.catalyst.util.DateTimeUtils
-+import org.apache.spark.sql.comet.CometLocalLimitExec
- import org.apache.spark.sql.execution.{LocalLimitExec, SimpleMode, SparkPlan}
- import org.apache.spark.sql.execution.command.ExplainCommand
- import org.apache.spark.sql.execution.streaming._
-@@ -1114,11 +1115,12 @@ class StreamSuite extends StreamTest {
-       val localLimits = execPlan.collect {
-         case l: LocalLimitExec => l
-         case l: StreamingLocalLimitExec => l
-+        case l: CometLocalLimitExec => l
-       }
- 
-       require(
-         localLimits.size == 1,
--        s"Cant verify local limit optimization with this plan:\n$execPlan")
-+        s"Cant verify local limit optimization ${localLimits.size} with this plan:\n$execPlan")
- 
-       if (expectStreamingLimit) {
-         assert(
-@@ -1126,7 +1128,8 @@ class StreamSuite extends StreamTest {
-           s"Local limit was not StreamingLocalLimitExec:\n$execPlan")
-       } else {
-         assert(
--          localLimits.head.isInstanceOf[LocalLimitExec],
-+          localLimits.head.isInstanceOf[LocalLimitExec] ||
-+            localLimits.head.isInstanceOf[CometLocalLimitExec],
-           s"Local limit was not LocalLimitExec:\n$execPlan")
-       }
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala
-index b4c4ec7acbf..20579284856 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala
-@@ -23,6 +23,7 @@ import org.apache.commons.io.FileUtils
- import org.scalatest.Assertions
- 
- import org.apache.spark.sql.catalyst.plans.physical.UnspecifiedDistribution
-+import org.apache.spark.sql.comet.CometHashAggregateExec
- import org.apache.spark.sql.execution.aggregate.BaseAggregateExec
- import org.apache.spark.sql.execution.streaming.{MemoryStream, StateStoreRestoreExec, StateStoreSaveExec}
- import org.apache.spark.sql.functions.count
-@@ -67,6 +68,7 @@ class StreamingAggregationDistributionSuite extends StreamTest
-         // verify aggregations in between, except partial aggregation
-         val allAggregateExecs = query.lastExecution.executedPlan.collect {
-           case a: BaseAggregateExec => a
-+          case c: CometHashAggregateExec => c.originalPlan
-         }
- 
-         val aggregateExecsWithoutPartialAgg = allAggregateExecs.filter {
-@@ -201,6 +203,7 @@ class StreamingAggregationDistributionSuite extends StreamTest
-         // verify aggregations in between, except partial aggregation
-         val allAggregateExecs = executedPlan.collect {
-           case a: BaseAggregateExec => a
-+          case c: CometHashAggregateExec => c.originalPlan
-         }
- 
-         val aggregateExecsWithoutPartialAgg = allAggregateExecs.filter {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
-index aad91601758..201083bd621 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
-@@ -31,7 +31,7 @@ import org.apache.spark.scheduler.ExecutorCacheTaskLocation
- import org.apache.spark.sql.{DataFrame, Row, SparkSession}
- import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression}
- import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.streaming.{MemoryStream, StatefulOperatorStateInfo, StreamingSymmetricHashJoinExec, StreamingSymmetricHashJoinHelper}
- import org.apache.spark.sql.execution.streaming.state.{RocksDBStateStoreProvider, StateStore, StateStoreProviderId}
- import org.apache.spark.sql.functions._
-@@ -619,14 +619,27 @@ class StreamingInnerJoinSuite extends StreamingJoinSuite {
- 
-         val numPartitions = spark.sqlContext.conf.getConf(SQLConf.SHUFFLE_PARTITIONS)
- 
--        assert(query.lastExecution.executedPlan.collect {
--          case j @ StreamingSymmetricHashJoinExec(_, _, _, _, _, _, _, _, _,
--            ShuffleExchangeExec(opA: HashPartitioning, _, _, _),
--            ShuffleExchangeExec(opB: HashPartitioning, _, _, _))
--              if partitionExpressionsColumns(opA.expressions) === Seq("a", "b")
--                && partitionExpressionsColumns(opB.expressions) === Seq("a", "b")
--                && opA.numPartitions == numPartitions && opB.numPartitions == numPartitions => j
--        }.size == 1)
-+        val join = query.lastExecution.executedPlan.collect {
-+          case j: StreamingSymmetricHashJoinExec => j
-+        }.head
-+        val opA = join.left.collect {
-+          case s: ShuffleExchangeLike
-+            if s.outputPartitioning.isInstanceOf[HashPartitioning] &&
-+              partitionExpressionsColumns(
-+                s.outputPartitioning
-+                  .asInstanceOf[HashPartitioning].expressions) === Seq("a", "b") =>
-+            s.outputPartitioning.asInstanceOf[HashPartitioning]
-+        }.head
-+        val opB = join.right.collect {
-+          case s: ShuffleExchangeLike
-+            if s.outputPartitioning.isInstanceOf[HashPartitioning] &&
-+              partitionExpressionsColumns(
-+                s.outputPartitioning
-+                  .asInstanceOf[HashPartitioning].expressions) === Seq("a", "b") =>
-+            s.outputPartitioning
-+              .asInstanceOf[HashPartitioning]
-+        }.head
-+        assert(opA.numPartitions == numPartitions && opB.numPartitions == numPartitions)
-       })
-   }
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
-index abe606ad9c1..2d930b64cca 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
-@@ -22,7 +22,7 @@ import java.util
- 
- import org.scalatest.BeforeAndAfter
- 
--import org.apache.spark.sql.{AnalysisException, Row, SaveMode}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, Row, SaveMode}
- import org.apache.spark.sql.catalyst.TableIdentifier
- import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
- import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
-@@ -327,7 +327,8 @@ class DataStreamTableAPISuite extends StreamTest with BeforeAndAfter {
-     }
-   }
- 
--  test("explain with table on DSv1 data source") {
-+  test("explain with table on DSv1 data source",
-+      IgnoreComet("Comet explain output is different")) {
-     val tblSourceName = "tbl_src"
-     val tblTargetName = "tbl_target"
-     val tblSourceQualified = s"default.$tblSourceName"
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-index e937173a590..18c0232014b 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-@@ -27,6 +27,7 @@ import scala.concurrent.duration._
- import scala.language.implicitConversions
- import scala.util.control.NonFatal
- 
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.scalactic.source.Position
- import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
-@@ -41,6 +42,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
- import org.apache.spark.sql.catalyst.plans.PlanTestBase
- import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
- import org.apache.spark.sql.catalyst.util._
-+import org.apache.spark.sql.comet._
- import org.apache.spark.sql.execution.FilterExec
- import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
- import org.apache.spark.sql.execution.datasources.DataSourceUtils
-@@ -126,7 +128,26 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
-         }
-       }
-     } else {
--      super.test(testName, testTags: _*)(testFun)
-+      if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
-+        ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
-+      } else {
-+        val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
-+        val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT
-+        val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION
-+        if (isCometEnabled && isNativeIcebergCompat &&
-+          testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
-+          ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
-+        } else if (isCometEnabled && isNativeDataFusion &&
-+          testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
-+          ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
-+        } else if (isCometEnabled && (isNativeDataFusion || isNativeIcebergCompat) &&
-+          testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
-+          ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
-+            testTags: _*)(testFun)
-+        } else {
-+          super.test(testName, testTags: _*)(testFun)
-+        }
-+      }
-     }
-   }
- 
-@@ -242,6 +263,29 @@ private[sql] trait SQLTestUtilsBase
-     protected override def _sqlContext: SQLContext = self.spark.sqlContext
-   }
- 
-+  /**
-+   * Whether Comet extension is enabled
-+   */
-+  protected def isCometEnabled: Boolean = SparkSession.isCometEnabled
-+
-+  /**
-+   * Whether to enable ansi mode This is only effective when
-+   * [[isCometEnabled]] returns true.
-+   */
-+  protected def enableCometAnsiMode: Boolean = {
-+    val v = System.getenv("ENABLE_COMET_ANSI_MODE")
-+    v != null && v.toBoolean
-+  }
-+
-+  /**
-+   * Whether Spark should only apply Comet scan optimization. This is only effective when
-+   * [[isCometEnabled]] returns true.
-+   */
-+  protected def isCometScanOnly: Boolean = {
-+    val v = System.getenv("ENABLE_COMET_SCAN_ONLY")
-+    v != null && v.toBoolean
-+  }
-+
-   protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
-     SparkSession.setActiveSession(spark)
-     super.withSQLConf(pairs: _*)(f)
-@@ -435,6 +479,8 @@ private[sql] trait SQLTestUtilsBase
-     val schema = df.schema
-     val withoutFilters = df.queryExecution.executedPlan.transform {
-       case FilterExec(_, child) => child
-+      case CometFilterExec(_, _, _, _, child, _) => child
-+      case CometProjectExec(_, _, _, _, CometFilterExec(_, _, _, _, child, _), _) => child
-     }
- 
-     spark.internalCreateDataFrame(withoutFilters.execute(), schema)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-index ed2e309fa07..71ba6533c9d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-@@ -74,6 +74,31 @@ trait SharedSparkSessionBase
-       // this rule may potentially block testing of other optimization rules such as
-       // ConstantPropagation etc.
-       .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)
-+    // Enable Comet if `ENABLE_COMET` environment variable is set
-+    if (isCometEnabled) {
-+      conf
-+        .set("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions")
-+        .set("spark.comet.enabled", "true")
-+
-+      if (!isCometScanOnly) {
-+        conf
-+          .set("spark.comet.exec.enabled", "true")
-+          .set("spark.shuffle.manager",
-+            "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
-+          .set("spark.comet.exec.shuffle.enabled", "true")
-+          .set("spark.comet.memoryOverhead", "10g")
-+      } else {
-+        conf
-+          .set("spark.comet.exec.enabled", "false")
-+          .set("spark.comet.exec.shuffle.enabled", "false")
-+      }
-+
-+      if (enableCometAnsiMode) {
-+        conf
-+          .set("spark.sql.ansi.enabled", "true")
-+          .set("spark.comet.ansi.enabled", "true")
-+      }
-+    }
-     conf.set(
-       StaticSQLConf.WAREHOUSE_PATH,
-       conf.get(StaticSQLConf.WAREHOUSE_PATH) + "/" + getClass.getCanonicalName)
-diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
-index c63c748953f..7edca9c93a6 100644
---- a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
-@@ -45,7 +45,7 @@ class SqlResourceWithActualMetricsSuite
-   import testImplicits._
- 
-   // Exclude nodes which may not have the metrics
--  val excludedNodes = List("WholeStageCodegen", "Project", "SerializeFromObject")
-+  val excludedNodes = List("WholeStageCodegen", "Project", "SerializeFromObject", "RowToColumnar")
- 
-   implicit val formats = new DefaultFormats {
-     override def dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
-diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala
-index 52abd248f3a..7a199931a08 100644
---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala
-+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala
-@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive
- 
- import org.apache.spark.sql._
- import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
-+import org.apache.spark.sql.comet._
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.hive.execution.HiveTableScanExec
-@@ -35,6 +36,9 @@ abstract class DynamicPartitionPruningHiveScanSuiteBase
-       case s: FileSourceScanExec => s.partitionFilters.collect {
-         case d: DynamicPruningExpression => d.child
-       }
-+      case s: CometScanExec => s.partitionFilters.collect {
-+        case d: DynamicPruningExpression => d.child
-+      }
-       case h: HiveTableScanExec => h.partitionPruningPred.collect {
-         case d: DynamicPruningExpression => d.child
-       }
-diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
-index dc8b184fcee..dd69a989d40 100644
---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
-+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
-@@ -660,7 +660,8 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
-         Row(3, 4, 4, 3, null) :: Nil)
-   }
- 
--  test("single distinct multiple columns set") {
-+  test("single distinct multiple columns set",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     checkAnswer(
-       spark.sql(
-         """
-diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-index 1d646f40b3e..7f2cdb8f061 100644
---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-@@ -53,25 +53,55 @@ object TestHive
-     new SparkContext(
-       System.getProperty("spark.sql.test.master", "local[1]"),
-       "TestSQLContext",
--      new SparkConf()
--        .set("spark.sql.test", "")
--        .set(SQLConf.CODEGEN_FALLBACK.key, "false")
--        .set(SQLConf.CODEGEN_FACTORY_MODE.key, CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
--        .set(HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key,
--          "org.apache.spark.sql.hive.execution.PairSerDe")
--        .set(WAREHOUSE_PATH.key, TestHiveContext.makeWarehouseDir().toURI.getPath)
--        // SPARK-8910
--        .set(UI_ENABLED, false)
--        .set(config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
--        // Hive changed the default of hive.metastore.disallow.incompatible.col.type.changes
--        // from false to true. For details, see the JIRA HIVE-12320 and HIVE-17764.
--        .set("spark.hadoop.hive.metastore.disallow.incompatible.col.type.changes", "false")
--        // Disable ConvertToLocalRelation for better test coverage. Test cases built on
--        // LocalRelation will exercise the optimization rules better by disabling it as
--        // this rule may potentially block testing of other optimization rules such as
--        // ConstantPropagation etc.
--        .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)))
-+      {
-+        val conf = new SparkConf()
-+          .set("spark.sql.test", "")
-+          .set(SQLConf.CODEGEN_FALLBACK.key, "false")
-+          .set(SQLConf.CODEGEN_FACTORY_MODE.key, CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
-+          .set(HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key,
-+            "org.apache.spark.sql.hive.execution.PairSerDe")
-+          .set(WAREHOUSE_PATH.key, TestHiveContext.makeWarehouseDir().toURI.getPath)
-+          // SPARK-8910
-+          .set(UI_ENABLED, false)
-+          .set(config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
-+          // Hive changed the default of hive.metastore.disallow.incompatible.col.type.changes
-+          // from false to true. For details, see the JIRA HIVE-12320 and HIVE-17764.
-+          .set("spark.hadoop.hive.metastore.disallow.incompatible.col.type.changes", "false")
-+          // Disable ConvertToLocalRelation for better test coverage. Test cases built on
-+          // LocalRelation will exercise the optimization rules better by disabling it as
-+          // this rule may potentially block testing of other optimization rules such as
-+          // ConstantPropagation etc.
-+          .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)
-+
-+        if (SparkSession.isCometEnabled) {
-+          conf
-+            .set("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions")
-+            .set("spark.comet.enabled", "true")
-+
-+          val v = System.getenv("ENABLE_COMET_SCAN_ONLY")
-+          if (v == null || !v.toBoolean) {
-+            conf
-+              .set("spark.comet.exec.enabled", "true")
-+              .set("spark.shuffle.manager",
-+                "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
-+              .set("spark.comet.exec.shuffle.enabled", "true")
-+          } else {
-+            conf
-+              .set("spark.comet.exec.enabled", "false")
-+              .set("spark.comet.exec.shuffle.enabled", "false")
-+          }
-+
-+          val a = System.getenv("ENABLE_COMET_ANSI_MODE")
-+          if (a != null && a.toBoolean) {
-+            conf
-+              .set("spark.sql.ansi.enabled", "true")
-+              .set("spark.comet.ansi.enabled", "true")
-+          }
-+        }
- 
-+        conf
-+      }
-+    ))
- 
- case class TestHiveVersion(hiveClient: HiveClient)
-   extends TestHiveContext(TestHive.sparkContext, hiveClient)
diff --git a/dev/diffs/3.5.5.diff b/dev/diffs/3.5.5.diff
deleted file mode 100644
index 75928a0773..0000000000
--- a/dev/diffs/3.5.5.diff
+++ /dev/null
@@ -1,3082 +0,0 @@
-diff --git a/pom.xml b/pom.xml
-index 9b009c3a42a..b134596c818 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -152,6 +152,8 @@
-     -->
-     <ivy.version>2.5.1</ivy.version>
-     <oro.version>2.0.8</oro.version>
-+    <spark.version.short>3.5</spark.version.short>
-+    <comet.version>0.9.0-SNAPSHOT</comet.version>
-     <!--
-     If you changes codahale.metrics.version, you also need to change
-     the link to metrics.dropwizard.io in docs/monitoring.md.
-@@ -2840,6 +2842,25 @@
-         <artifactId>okio</artifactId>
-         <version>${okio.version}</version>
-       </dependency>
-+      <dependency>
-+        <groupId>org.apache.datafusion</groupId>
-+        <artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
-+        <version>${comet.version}</version>
-+        <exclusions>
-+          <exclusion>
-+            <groupId>org.apache.spark</groupId>
-+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
-+          </exclusion>
-+          <exclusion>
-+            <groupId>org.apache.spark</groupId>
-+            <artifactId>spark-core_${scala.binary.version}</artifactId>
-+          </exclusion>
-+          <exclusion>
-+            <groupId>org.apache.spark</groupId>
-+            <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
-+          </exclusion>
-+        </exclusions>
-+      </dependency>
-     </dependencies>
-   </dependencyManagement>
- 
-diff --git a/sql/core/pom.xml b/sql/core/pom.xml
-index 91860dff2b4..3277434fb7d 100644
---- a/sql/core/pom.xml
-+++ b/sql/core/pom.xml
-@@ -77,6 +77,10 @@
-       <groupId>org.apache.spark</groupId>
-       <artifactId>spark-tags_${scala.binary.version}</artifactId>
-     </dependency>
-+    <dependency>
-+      <groupId>org.apache.datafusion</groupId>
-+      <artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
-+    </dependency>
- 
-     <!--
-       This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
-diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
-index 27ae10b3d59..78e69902dfd 100644
---- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
-+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
-@@ -1353,6 +1353,14 @@ object SparkSession extends Logging {
-     }
-   }
- 
-+  private def loadCometExtension(sparkContext: SparkContext): Seq[String] = {
-+    if (sparkContext.getConf.getBoolean("spark.comet.enabled", isCometEnabled)) {
-+      Seq("org.apache.comet.CometSparkSessionExtensions")
-+    } else {
-+      Seq.empty
-+    }
-+  }
-+
-   /**
-    * Initialize extensions specified in [[StaticSQLConf]]. The classes will be applied to the
-    * extensions passed into this function.
-@@ -1362,6 +1370,7 @@ object SparkSession extends Logging {
-       extensions: SparkSessionExtensions): SparkSessionExtensions = {
-     val extensionConfClassNames = sparkContext.getConf.get(StaticSQLConf.SPARK_SESSION_EXTENSIONS)
-       .getOrElse(Seq.empty)
-+    val extensionClassNames = extensionConfClassNames ++ loadCometExtension(sparkContext)
-     extensionConfClassNames.foreach { extensionConfClassName =>
-       try {
-         val extensionConfClass = Utils.classForName(extensionConfClassName)
-@@ -1396,4 +1405,12 @@ object SparkSession extends Logging {
-       }
-     }
-   }
-+
-+  /**
-+   * Whether Comet extension is enabled
-+   */
-+  def isCometEnabled: Boolean = {
-+    val v = System.getenv("ENABLE_COMET")
-+    v == null || v.toBoolean
-+  }
- }
-diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
-index db587dd9868..aac7295a53d 100644
---- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
-+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.annotation.DeveloperApi
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
- import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
- import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
-@@ -67,6 +68,7 @@ private[execution] object SparkPlanInfo {
-     // dump the file scan metadata (e.g file path) to event log
-     val metadata = plan match {
-       case fileScan: FileSourceScanExec => fileScan.metadata
-+      case cometScan: CometScanExec => cometScan.metadata
-       case _ => Map[String, String]()
-     }
-     new SparkPlanInfo(
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
-index 7aef901da4f..f3d6e18926d 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
-@@ -2,3 +2,4 @@
- 
- --SET spark.sql.adaptive.enabled=true
- --SET spark.sql.maxMetadataStringLength = 500
-+--SET spark.comet.enabled = false
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
-index eeb2180f7a5..afd1b5ec289 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-cbo.sql
-@@ -1,5 +1,6 @@
- --SET spark.sql.cbo.enabled=true
- --SET spark.sql.maxMetadataStringLength = 500
-+--SET spark.comet.enabled = false
- 
- CREATE TABLE explain_temp1(a INT, b INT) USING PARQUET;
- CREATE TABLE explain_temp2(c INT, d INT) USING PARQUET;
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain.sql b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
-index 698ca009b4f..57d774a3617 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/explain.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
-@@ -1,6 +1,7 @@
- --SET spark.sql.codegen.wholeStage = true
- --SET spark.sql.adaptive.enabled = false
- --SET spark.sql.maxMetadataStringLength = 500
-+--SET spark.comet.enabled = false
- 
- -- Test tables
- CREATE table  explain_temp1 (key int, val int) USING PARQUET;
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql
-index 1152d77da0c..f77493f690b 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part1.sql
-@@ -7,6 +7,9 @@
- 
- -- avoid bit-exact output here because operations may not be bit-exact.
- -- SET extra_float_digits = 0;
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- 
- -- Test aggregate operator with codegen on and off.
- --CONFIG_DIM1 spark.sql.codegen.wholeStage=true
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
-index 41fd4de2a09..44cd244d3b0 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
-@@ -5,6 +5,9 @@
- -- AGGREGATES [Part 3]
- -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/aggregates.sql#L352-L605
- 
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- -- Test aggregate operator with codegen on and off.
- --CONFIG_DIM1 spark.sql.codegen.wholeStage=true
- --CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql
-index fac23b4a26f..2b73732c33f 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/int8.sql
-@@ -1,6 +1,10 @@
- --
- -- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
- --
-+
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- --
- -- INT8
- -- Test int8 64-bit integers.
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
-index 0efe0877e9b..423d3b3d76d 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
-@@ -1,6 +1,10 @@
- --
- -- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
- --
-+
-+-- Disable Comet exec due to floating point precision difference
-+--SET spark.comet.exec.enabled = false
-+
- --
- -- SELECT_HAVING
- -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/select_having.sql
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
-index 9815cb816c9..95b5f9992b0 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
-@@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants
- import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, RDDScanExec, SparkPlan}
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQEPropagateEmptyRelation}
- import org.apache.spark.sql.execution.columnar._
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -519,7 +519,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
-       df.collect()
-     }
-     assert(
--      collect(df.queryExecution.executedPlan) { case e: ShuffleExchangeExec => e }.size == expected)
-+      collect(df.queryExecution.executedPlan) {
-+        case _: ShuffleExchangeLike => 1 }.size == expected)
-   }
- 
-   test("A cached table preserves the partitioning and ordering of its cached SparkPlan") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
-index 5a8681aed97..da9d25e2eb4 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
-@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.plans.logical.Expand
- import org.apache.spark.sql.execution.WholeStageCodegenExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.expressions.Window
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -793,7 +793,7 @@ class DataFrameAggregateSuite extends QueryTest
-       assert(objHashAggPlans.nonEmpty)
- 
-       val exchangePlans = collect(aggPlan) {
--        case shuffle: ShuffleExchangeExec => shuffle
-+        case shuffle: ShuffleExchangeLike => shuffle
-       }
-       assert(exchangePlans.length == 1)
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
-index 56e9520fdab..917932336df 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
-@@ -435,7 +435,9 @@ class DataFrameJoinSuite extends QueryTest
- 
-     withTempDatabase { dbName =>
-       withTable(table1Name, table2Name) {
--        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-+        withSQLConf(
-+            SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-+            "spark.comet.enabled" -> "false") {
-           spark.range(50).write.saveAsTable(s"$dbName.$table1Name")
-           spark.range(100).write.saveAsTable(s"$dbName.$table2Name")
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-index 7ee18df3756..64f01a68048 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-@@ -40,11 +40,12 @@ import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
- import org.apache.spark.sql.catalyst.parser.ParseException
- import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LeafNode, LocalRelation, LogicalPlan, OneRowRelation, Statistics}
- import org.apache.spark.sql.catalyst.util.DateTimeUtils
-+import org.apache.spark.sql.comet.CometBroadcastExchangeExec
- import org.apache.spark.sql.connector.FakeV2Provider
- import org.apache.spark.sql.execution.{FilterExec, LogicalRDD, QueryExecution, SortExec, WholeStageCodegenExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.aggregate.HashAggregateExec
--import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
-+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.expressions.{Aggregator, Window}
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -2006,7 +2007,7 @@ class DataFrameSuite extends QueryTest
-           fail("Should not have back to back Aggregates")
-         }
-         atFirstAgg = true
--      case e: ShuffleExchangeExec => atFirstAgg = false
-+      case e: ShuffleExchangeLike => atFirstAgg = false
-       case _ =>
-     }
-   }
-@@ -2330,7 +2331,7 @@ class DataFrameSuite extends QueryTest
-       checkAnswer(join, df)
-       assert(
-         collect(join.queryExecution.executedPlan) {
--          case e: ShuffleExchangeExec => true }.size === 1)
-+          case _: ShuffleExchangeLike => true }.size === 1)
-       assert(
-         collect(join.queryExecution.executedPlan) { case e: ReusedExchangeExec => true }.size === 1)
-       val broadcasted = broadcast(join)
-@@ -2338,10 +2339,12 @@ class DataFrameSuite extends QueryTest
-       checkAnswer(join2, df)
-       assert(
-         collect(join2.queryExecution.executedPlan) {
--          case e: ShuffleExchangeExec => true }.size == 1)
-+          case _: ShuffleExchangeLike => true }.size == 1)
-       assert(
-         collect(join2.queryExecution.executedPlan) {
--          case e: BroadcastExchangeExec => true }.size === 1)
-+          case e: BroadcastExchangeExec => true
-+          case _: CometBroadcastExchangeExec => true
-+        }.size === 1)
-       assert(
-         collect(join2.queryExecution.executedPlan) { case e: ReusedExchangeExec => true }.size == 4)
-     }
-@@ -2901,7 +2904,7 @@ class DataFrameSuite extends QueryTest
- 
-     // Assert that no extra shuffle introduced by cogroup.
-     val exchanges = collect(df3.queryExecution.executedPlan) {
--      case h: ShuffleExchangeExec => h
-+      case h: ShuffleExchangeLike => h
-     }
-     assert(exchanges.size == 2)
-   }
-@@ -3350,7 +3353,8 @@ class DataFrameSuite extends QueryTest
-     assert(df2.isLocal)
-   }
- 
--  test("SPARK-35886: PromotePrecision should be subexpr replaced") {
-+  test("SPARK-35886: PromotePrecision should be subexpr replaced",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     withTable("tbl") {
-       sql(
-         """
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-index 47a311c71d5..342e71cfdd4 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-@@ -24,8 +24,9 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression
- import org.apache.spark.sql.catalyst.optimizer.TransposeWindow
- import org.apache.spark.sql.catalyst.plans.logical.{Window => LogicalWindow}
- import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
-+import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
--import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, Exchange, ShuffleExchangeExec}
-+import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, Exchange, ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.window.WindowExec
- import org.apache.spark.sql.expressions.{Aggregator, MutableAggregationBuffer, UserDefinedAggregateFunction, Window}
- import org.apache.spark.sql.functions._
-@@ -1187,10 +1188,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest
-     }
- 
-     def isShuffleExecByRequirement(
--        plan: ShuffleExchangeExec,
-+        plan: ShuffleExchangeLike,
-         desiredClusterColumns: Seq[String]): Boolean = plan match {
-       case ShuffleExchangeExec(op: HashPartitioning, _, ENSURE_REQUIREMENTS, _) =>
-         partitionExpressionsColumns(op.expressions) === desiredClusterColumns
-+      case CometShuffleExchangeExec(op: HashPartitioning, _, _, ENSURE_REQUIREMENTS, _, _) =>
-+        partitionExpressionsColumns(op.expressions) === desiredClusterColumns
-       case _ => false
-     }
- 
-@@ -1213,7 +1216,7 @@ class DataFrameWindowFunctionsSuite extends QueryTest
-       val shuffleByRequirement = windowed.queryExecution.executedPlan.exists {
-         case w: WindowExec =>
-           w.child.exists {
--            case s: ShuffleExchangeExec => isShuffleExecByRequirement(s, Seq("key1", "key2"))
-+            case s: ShuffleExchangeLike => isShuffleExecByRequirement(s, Seq("key1", "key2"))
-             case _ => false
-           }
-         case _ => false
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
-index f32b32ffc5a..447d7c6416e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
-@@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
- import org.apache.spark.sql.catalyst.util.sideBySide
- import org.apache.spark.sql.execution.{LogicalRDD, RDDScanExec, SQLExecution}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
--import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec}
-+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.streaming.MemoryStream
- import org.apache.spark.sql.expressions.UserDefinedFunction
- import org.apache.spark.sql.functions._
-@@ -2288,7 +2288,7 @@ class DatasetSuite extends QueryTest
- 
-     // Assert that no extra shuffle introduced by cogroup.
-     val exchanges = collect(df3.queryExecution.executedPlan) {
--      case h: ShuffleExchangeExec => h
-+      case h: ShuffleExchangeLike => h
-     }
-     assert(exchanges.size == 2)
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
-index f33432ddb6f..0e1499a24ca 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
-@@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen
- import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
- import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
- import org.apache.spark.sql.catalyst.plans.ExistenceJoin
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.connector.catalog.{InMemoryTableCatalog, InMemoryTableWithV2FilterCatalog}
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.adaptive._
-@@ -262,6 +263,9 @@ abstract class DynamicPartitionPruningSuiteBase
-       case s: BatchScanExec => s.runtimeFilters.collect {
-         case d: DynamicPruningExpression => d.child
-       }
-+      case s: CometScanExec => s.partitionFilters.collect {
-+        case d: DynamicPruningExpression => d.child
-+      }
-       case _ => Nil
-     }
-   }
-@@ -1027,7 +1031,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("avoid reordering broadcast join keys to match input hash partitioning") {
-+  test("avoid reordering broadcast join keys to match input hash partitioning",
-+    IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-       withTable("large", "dimTwo", "dimThree") {
-@@ -1215,7 +1220,8 @@ abstract class DynamicPartitionPruningSuiteBase
-   }
- 
-   test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
--    "canonicalization and exchange reuse") {
-+    "canonicalization and exchange reuse",
-+    IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
-         val df = sql(
-@@ -1423,7 +1429,8 @@ abstract class DynamicPartitionPruningSuiteBase
-     }
-   }
- 
--  test("SPARK-34637: DPP side broadcast query stage is created firstly") {
-+  test("SPARK-34637: DPP side broadcast query stage is created firstly",
-+    IgnoreComet("TODO: https://github.com/apache/datafusion-comet/issues/1839")) {
-     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
-       val df = sql(
-         """ WITH v as (
-@@ -1729,6 +1736,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat
-               case s: BatchScanExec =>
-                 // we use f1 col for v2 tables due to schema pruning
-                 s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))
-+              case s: CometScanExec =>
-+                s.output.exists(_.exists(_.argString(maxFields = 100).contains("fid")))
-               case _ => false
-             }
-           assert(scanOption.isDefined)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-index a206e97c353..fea1149b67d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-@@ -467,7 +467,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
-     }
-   }
- 
--  test("Explain formatted output for scan operator for datasource V2") {
-+  test("Explain formatted output for scan operator for datasource V2",
-+      IgnoreComet("Comet explain output is different")) {
-     withTempDir { dir =>
-       Seq("parquet", "orc", "csv", "json").foreach { fmt =>
-         val basePath = dir.getCanonicalPath + "/" + fmt
-@@ -545,7 +546,9 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
-   }
- }
- 
--class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuite {
-+// Ignored when Comet is enabled. Comet changes expected query plans.
-+class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuite
-+    with IgnoreCometSuite {
-   import testImplicits._
- 
-   test("SPARK-35884: Explain Formatted") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-index 93275487f29..01e5c601763 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-@@ -23,6 +23,7 @@ import java.nio.file.{Files, StandardOpenOption}
- 
- import scala.collection.mutable
- 
-+import org.apache.comet.CometConf
- import org.apache.hadoop.conf.Configuration
- import org.apache.hadoop.fs.{LocalFileSystem, Path}
- 
-@@ -33,6 +34,7 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GreaterTha
- import org.apache.spark.sql.catalyst.expressions.IntegralLiteralTestUtils.{negativeInt, positiveInt}
- import org.apache.spark.sql.catalyst.plans.logical.Filter
- import org.apache.spark.sql.catalyst.types.DataTypeUtils
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec, CometScanExec, CometSortMergeJoinExec}
- import org.apache.spark.sql.execution.{FileSourceScanLike, SimpleMode}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.datasources.FilePartition
-@@ -250,6 +252,8 @@ class FileBasedDataSourceSuite extends QueryTest
-               case "" => "_LEGACY_ERROR_TEMP_2062"
-               case _ => "_LEGACY_ERROR_TEMP_2055"
-             }
-+            // native_datafusion Parquet scan cannot throw a SparkFileNotFoundException
-+            assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_DATAFUSION)
-             checkErrorMatchPVals(
-               exception = intercept[SparkException] {
-                 testIgnoreMissingFiles(options)
-@@ -955,6 +959,7 @@ class FileBasedDataSourceSuite extends QueryTest
-             assert(bJoinExec.isEmpty)
-             val smJoinExec = collect(joinedDF.queryExecution.executedPlan) {
-               case smJoin: SortMergeJoinExec => smJoin
-+              case smJoin: CometSortMergeJoinExec => smJoin
-             }
-             assert(smJoinExec.nonEmpty)
-           }
-@@ -1015,6 +1020,7 @@ class FileBasedDataSourceSuite extends QueryTest
- 
-           val fileScan = df.queryExecution.executedPlan collectFirst {
-             case BatchScanExec(_, f: FileScan, _, _, _, _) => f
-+            case CometBatchScanExec(BatchScanExec(_, f: FileScan, _, _, _, _), _) => f
-           }
-           assert(fileScan.nonEmpty)
-           assert(fileScan.get.partitionFilters.nonEmpty)
-@@ -1056,6 +1062,7 @@ class FileBasedDataSourceSuite extends QueryTest
- 
-           val fileScan = df.queryExecution.executedPlan collectFirst {
-             case BatchScanExec(_, f: FileScan, _, _, _, _) => f
-+            case CometBatchScanExec(BatchScanExec(_, f: FileScan, _, _, _, _), _) => f
-           }
-           assert(fileScan.nonEmpty)
-           assert(fileScan.get.partitionFilters.isEmpty)
-@@ -1240,6 +1247,9 @@ class FileBasedDataSourceSuite extends QueryTest
-           val filters = df.queryExecution.executedPlan.collect {
-             case f: FileSourceScanLike => f.dataFilters
-             case b: BatchScanExec => b.scan.asInstanceOf[FileScan].dataFilters
-+            case b: CometScanExec => b.dataFilters
-+            case b: CometNativeScanExec => b.dataFilters
-+            case b: CometBatchScanExec => b.scan.asInstanceOf[FileScan].dataFilters
-           }.flatten
-           assert(filters.contains(GreaterThan(scan.logicalPlan.output.head, Literal(5L))))
-         }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-new file mode 100644
-index 00000000000..5691536c114
---- /dev/null
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-@@ -0,0 +1,45 @@
-+/*
-+ * Licensed to the Apache Software Foundation (ASF) under one or more
-+ * contributor license agreements.  See the NOTICE file distributed with
-+ * this work for additional information regarding copyright ownership.
-+ * The ASF licenses this file to You under the Apache License, Version 2.0
-+ * (the "License"); you may not use this file except in compliance with
-+ * the License.  You may obtain a copy of the License at
-+ *
-+ *    http://www.apache.org/licenses/LICENSE-2.0
-+ *
-+ * Unless required by applicable law or agreed to in writing, software
-+ * distributed under the License is distributed on an "AS IS" BASIS,
-+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+ * See the License for the specific language governing permissions and
-+ * limitations under the License.
-+ */
-+
-+package org.apache.spark.sql
-+
-+import org.scalactic.source.Position
-+import org.scalatest.Tag
-+
-+import org.apache.spark.sql.test.SQLTestUtils
-+
-+/**
-+ * Tests with this tag will be ignored when Comet is enabled (e.g., via `ENABLE_COMET`).
-+ */
-+case class IgnoreComet(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeIcebergCompat(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeDataFusion(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeScan(reason: String) extends Tag("DisableComet")
-+
-+/**
-+ * Helper trait that disables Comet for all tests regardless of default config values.
-+ */
-+trait IgnoreCometSuite extends SQLTestUtils {
-+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
-+    (implicit pos: Position): Unit = {
-+    if (isCometEnabled) {
-+      ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
-+    } else {
-+      super.test(testName, testTags: _*)(testFun)
-+    }
-+  }
-+}
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
-index 7af826583bd..3c3def1eb67 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
-@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide
- import org.apache.spark.sql.catalyst.plans.PlanTest
- import org.apache.spark.sql.catalyst.plans.logical._
- import org.apache.spark.sql.catalyst.rules.RuleExecutor
-+import org.apache.spark.sql.comet.{CometHashJoinExec, CometSortMergeJoinExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.joins._
- import org.apache.spark.sql.internal.SQLConf
-@@ -362,6 +363,7 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
-     val executedPlan = df.queryExecution.executedPlan
-     val shuffleHashJoins = collect(executedPlan) {
-       case s: ShuffledHashJoinExec => s
-+      case c: CometHashJoinExec => c.originalPlan.asInstanceOf[ShuffledHashJoinExec]
-     }
-     assert(shuffleHashJoins.size == 1)
-     assert(shuffleHashJoins.head.buildSide == buildSide)
-@@ -371,6 +373,7 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
-     val executedPlan = df.queryExecution.executedPlan
-     val shuffleMergeJoins = collect(executedPlan) {
-       case s: SortMergeJoinExec => s
-+      case c: CometSortMergeJoinExec => c
-     }
-     assert(shuffleMergeJoins.size == 1)
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-index 4d256154c85..66a5473852d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
-@@ -31,7 +31,8 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
- import org.apache.spark.sql.catalyst.expressions.{Ascending, GenericRow, SortOrder}
- import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
- import org.apache.spark.sql.catalyst.plans.logical.{Filter, HintInfo, Join, JoinHint, NO_BROADCAST_AND_REPLICATION}
--import org.apache.spark.sql.execution.{BinaryExecNode, FilterExec, ProjectExec, SortExec, SparkPlan, WholeStageCodegenExec}
-+import org.apache.spark.sql.comet._
-+import org.apache.spark.sql.execution.{BinaryExecNode, ColumnarToRowExec, FilterExec, InputAdapter, ProjectExec, SortExec, SparkPlan, WholeStageCodegenExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.joins._
-@@ -802,7 +803,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-     }
-   }
- 
--  test("test SortMergeJoin (with spill)") {
-+  test("test SortMergeJoin (with spill)",
-+      IgnoreComet("TODO: Comet SMJ doesn't support spill yet")) {
-     withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1",
-       SQLConf.SORT_MERGE_JOIN_EXEC_BUFFER_IN_MEMORY_THRESHOLD.key -> "0",
-       SQLConf.SORT_MERGE_JOIN_EXEC_BUFFER_SPILL_THRESHOLD.key -> "1") {
-@@ -928,10 +930,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-       val physical = df.queryExecution.sparkPlan
-       val physicalJoins = physical.collect {
-         case j: SortMergeJoinExec => j
-+        case j: CometSortMergeJoinExec => j.originalPlan.asInstanceOf[SortMergeJoinExec]
-       }
-       val executed = df.queryExecution.executedPlan
-       val executedJoins = collect(executed) {
-         case j: SortMergeJoinExec => j
-+        case j: CometSortMergeJoinExec => j.originalPlan.asInstanceOf[SortMergeJoinExec]
-       }
-       // This only applies to the above tested queries, in which a child SortMergeJoin always
-       // contains the SortOrder required by its parent SortMergeJoin. Thus, SortExec should never
-@@ -1177,9 +1181,11 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-       val plan = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2", joinType)
-         .groupBy($"k1").count()
-         .queryExecution.executedPlan
--      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: ShuffledHashJoinExec | _: CometHashJoinExec => true }.size === 1)
-       // No extra shuffle before aggregate
--      assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 2)
-+      assert(collect(plan) {
-+        case _: ShuffleExchangeLike => true }.size === 2)
-     })
-   }
- 
-@@ -1196,10 +1202,11 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-         .join(df4.hint("SHUFFLE_MERGE"), $"k1" === $"k4", joinType)
-         .queryExecution
-         .executedPlan
--      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 2)
-+      assert(collect(plan) {
-+        case _: SortMergeJoinExec | _: CometSortMergeJoinExec => true }.size === 2)
-       assert(collect(plan) { case _: BroadcastHashJoinExec => true }.size === 1)
-       // No extra sort before last sort merge join
--      assert(collect(plan) { case _: SortExec => true }.size === 3)
-+      assert(collect(plan) { case _: SortExec | _: CometSortExec => true }.size === 3)
-     })
- 
-     // Test shuffled hash join
-@@ -1209,10 +1216,13 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-         .join(df4.hint("SHUFFLE_MERGE"), $"k1" === $"k4", joinType)
-         .queryExecution
-         .executedPlan
--      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 2)
--      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: SortMergeJoinExec | _: CometSortMergeJoinExec => true }.size === 2)
-+      assert(collect(plan) {
-+        case _: ShuffledHashJoinExec | _: CometHashJoinExec => true }.size === 1)
-       // No extra sort before last sort merge join
--      assert(collect(plan) { case _: SortExec => true }.size === 3)
-+      assert(collect(plan) {
-+        case _: SortExec | _: CometSortExec => true }.size === 3)
-     })
-   }
- 
-@@ -1303,12 +1313,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-     inputDFs.foreach { case (df1, df2, joinExprs) =>
-       val smjDF = df1.join(df2.hint("SHUFFLE_MERGE"), joinExprs, "full")
-       assert(collect(smjDF.queryExecution.executedPlan) {
--        case _: SortMergeJoinExec => true }.size === 1)
-+        case _: SortMergeJoinExec | _: CometSortMergeJoinExec => true }.size === 1)
-       val smjResult = smjDF.collect()
- 
-       val shjDF = df1.join(df2.hint("SHUFFLE_HASH"), joinExprs, "full")
-       assert(collect(shjDF.queryExecution.executedPlan) {
--        case _: ShuffledHashJoinExec => true }.size === 1)
-+        case _: ShuffledHashJoinExec | _: CometHashJoinExec => true }.size === 1)
-       // Same result between shuffled hash join and sort merge join
-       checkAnswer(shjDF, smjResult)
-     }
-@@ -1367,12 +1377,14 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val smjDF = df1.hint("SHUFFLE_MERGE").join(df2, joinExprs, "leftouter")
-           assert(collect(smjDF.queryExecution.executedPlan) {
-             case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-           }.size === 1)
-           val smjResult = smjDF.collect()
- 
-           val shjDF = df1.hint("SHUFFLE_HASH").join(df2, joinExprs, "leftouter")
-           assert(collect(shjDF.queryExecution.executedPlan) {
-             case _: ShuffledHashJoinExec => true
-+            case _: CometHashJoinExec => true
-           }.size === 1)
-           // Same result between shuffled hash join and sort merge join
-           checkAnswer(shjDF, smjResult)
-@@ -1383,12 +1395,14 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val smjDF = df2.join(df1.hint("SHUFFLE_MERGE"), joinExprs, "rightouter")
-           assert(collect(smjDF.queryExecution.executedPlan) {
-             case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-           }.size === 1)
-           val smjResult = smjDF.collect()
- 
-           val shjDF = df2.join(df1.hint("SHUFFLE_HASH"), joinExprs, "rightouter")
-           assert(collect(shjDF.queryExecution.executedPlan) {
-             case _: ShuffledHashJoinExec => true
-+            case _: CometHashJoinExec => true
-           }.size === 1)
-           // Same result between shuffled hash join and sort merge join
-           checkAnswer(shjDF, smjResult)
-@@ -1432,13 +1446,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-         assert(shjCodegenDF.queryExecution.executedPlan.collect {
-           case WholeStageCodegenExec(_ : ShuffledHashJoinExec) => true
-           case WholeStageCodegenExec(ProjectExec(_, _ : ShuffledHashJoinExec)) => true
-+          case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(_: CometHashJoinExec))) =>
-+            true
-+          case WholeStageCodegenExec(ColumnarToRowExec(
-+            InputAdapter(CometProjectExec(_, _, _, _, _: CometHashJoinExec, _)))) => true
-+          case _: CometHashJoinExec => true
-         }.size === 1)
-         checkAnswer(shjCodegenDF, Seq.empty)
- 
-         withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
-           val shjNonCodegenDF = df1.join(df2.hint("SHUFFLE_HASH"), $"k1" === $"k2", joinType)
-           assert(shjNonCodegenDF.queryExecution.executedPlan.collect {
--            case _: ShuffledHashJoinExec => true }.size === 1)
-+            case _: ShuffledHashJoinExec => true
-+            case _: CometHashJoinExec => true
-+          }.size === 1)
-           checkAnswer(shjNonCodegenDF, Seq.empty)
-         }
-       }
-@@ -1486,7 +1507,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val plan = sql(getAggQuery(selectExpr, joinType)).queryExecution.executedPlan
-           assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
-           // Have shuffle before aggregation
--          assert(collect(plan) { case _: ShuffleExchangeExec => true }.size === 1)
-+          assert(collect(plan) {
-+            case _: ShuffleExchangeLike => true }.size === 1)
-       }
- 
-       def getJoinQuery(selectExpr: String, joinType: String): String = {
-@@ -1515,9 +1537,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           }
-           val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
-           assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
--          assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
-+          assert(collect(plan) {
-+            case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-+          }.size === 3)
-           // No extra sort on left side before last sort merge join
--          assert(collect(plan) { case _: SortExec => true }.size === 5)
-+          assert(collect(plan) { case _: SortExec | _: CometSortExec => true }.size === 5)
-       }
- 
-       // Test output ordering is not preserved
-@@ -1526,9 +1551,12 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           val selectExpr = "/*+ BROADCAST(left_t) */ k1 as k0"
-           val plan = sql(getJoinQuery(selectExpr, joinType)).queryExecution.executedPlan
-           assert(collect(plan) { case _: BroadcastNestedLoopJoinExec => true }.size === 1)
--          assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 3)
-+          assert(collect(plan) {
-+            case _: SortMergeJoinExec => true
-+            case _: CometSortMergeJoinExec => true
-+          }.size === 3)
-           // Have sort on left side before last sort merge join
--          assert(collect(plan) { case _: SortExec => true }.size === 6)
-+          assert(collect(plan) { case _: SortExec | _: CometSortExec => true }.size === 6)
-       }
- 
-       // Test singe partition
-@@ -1538,7 +1566,8 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-            |FROM range(0, 10, 1, 1) t1 FULL OUTER JOIN range(0, 10, 1, 1) t2
-            |""".stripMargin)
-       val plan = fullJoinDF.queryExecution.executedPlan
--      assert(collect(plan) { case _: ShuffleExchangeExec => true}.size == 1)
-+      assert(collect(plan) {
-+        case _: ShuffleExchangeLike => true}.size == 1)
-       checkAnswer(fullJoinDF, Row(100))
-     }
-   }
-@@ -1583,6 +1612,9 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
-           Seq(semiJoinDF, antiJoinDF).foreach { df =>
-             assert(collect(df.queryExecution.executedPlan) {
-               case j: ShuffledHashJoinExec if j.ignoreDuplicatedKey == ignoreDuplicatedKey => true
-+              case j: CometHashJoinExec
-+                if j.originalPlan.asInstanceOf[ShuffledHashJoinExec].ignoreDuplicatedKey ==
-+                  ignoreDuplicatedKey => true
-             }.size == 1)
-           }
-       }
-@@ -1627,14 +1659,20 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
- 
-   test("SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)") {
-     def check(plan: SparkPlan): Unit = {
--      assert(collect(plan) { case _: SortMergeJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: SortMergeJoinExec => true
-+        case _: CometSortMergeJoinExec => true
-+      }.size === 1)
-     }
-     dupStreamSideColTest("MERGE", check)
-   }
- 
-   test("SPARK-43113: Full outer join with duplicate stream-side references in condition (SHJ)") {
-     def check(plan: SparkPlan): Unit = {
--      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
-+      assert(collect(plan) {
-+        case _: ShuffledHashJoinExec => true
-+        case _: CometHashJoinExec => true
-+      }.size === 1)
-     }
-     dupStreamSideColTest("SHUFFLE_HASH", check)
-   }
-@@ -1770,7 +1808,8 @@ class ThreadLeakInSortMergeJoinSuite
-       sparkConf.set(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD, 20))
-   }
- 
--  test("SPARK-47146: thread leak when doing SortMergeJoin (with spill)") {
-+  test("SPARK-47146: thread leak when doing SortMergeJoin (with spill)",
-+    IgnoreComet("Comet does not support spilling")) {
- 
-     withSQLConf(
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
-index c26757c9cff..d55775f09d7 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala
-@@ -69,7 +69,7 @@ import org.apache.spark.tags.ExtendedSQLTest
-  * }}}
-  */
- // scalastyle:on line.size.limit
--trait PlanStabilitySuite extends DisableAdaptiveExecutionSuite {
-+trait PlanStabilitySuite extends DisableAdaptiveExecutionSuite with IgnoreCometSuite {
- 
-   protected val baseResourcePath = {
-     // use the same way as `SQLQueryTestSuite` to get the resource path
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
-index 793a0da6a86..6ccb9d62582 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
-@@ -1521,7 +1521,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
-     checkAnswer(sql("select -0.001"), Row(BigDecimal("-0.001")))
-   }
- 
--  test("external sorting updates peak execution memory") {
-+  test("external sorting updates peak execution memory",
-+    IgnoreComet("TODO: native CometSort does not update peak execution memory")) {
-     AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") {
-       sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect()
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-index 8b4ac474f87..3f79f20822f 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-@@ -223,6 +223,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     withSession(extensions) { session =>
-       session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true)
-       session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1")
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      session.conf.set("spark.comet.enabled", false)
-       assert(session.sessionState.columnarRules.contains(
-         MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
-       import session.sqlContext.implicits._
-@@ -281,6 +283,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     }
-     withSession(extensions) { session =>
-       session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE)
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      session.conf.set("spark.comet.enabled", false)
-       assert(session.sessionState.columnarRules.contains(
-         MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
-       import session.sqlContext.implicits._
-@@ -319,6 +323,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     val session = SparkSession.builder()
-       .master("local[1]")
-       .config(COLUMN_BATCH_SIZE.key, 2)
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      .config("spark.comet.enabled", false)
-       .withExtensions { extensions =>
-         extensions.injectColumnar(session =>
-           MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
-index fa1a64460fc..1d2e215d6a3 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
-@@ -17,6 +17,8 @@
- 
- package org.apache.spark.sql
- 
-+import org.apache.comet.CometConf
-+
- import org.apache.spark.{SPARK_DOC_ROOT, SparkIllegalArgumentException, SparkRuntimeException}
- import org.apache.spark.sql.catalyst.expressions.Cast._
- import org.apache.spark.sql.execution.FormattedMode
-@@ -178,29 +180,31 @@ class StringFunctionsSuite extends QueryTest with SharedSparkSession {
-   }
- 
-   test("string regex_replace / regex_extract") {
--    val df = Seq(
--      ("100-200", "(\\d+)-(\\d+)", "300"),
--      ("100-200", "(\\d+)-(\\d+)", "400"),
--      ("100-200", "(\\d+)", "400")).toDF("a", "b", "c")
-+    withSQLConf(CometConf.COMET_REGEXP_ALLOW_INCOMPATIBLE.key -> "true") {
-+      val df = Seq(
-+        ("100-200", "(\\d+)-(\\d+)", "300"),
-+        ("100-200", "(\\d+)-(\\d+)", "400"),
-+        ("100-200", "(\\d+)", "400")).toDF("a", "b", "c")
- 
--    checkAnswer(
--      df.select(
--        regexp_replace($"a", "(\\d+)", "num"),
--        regexp_replace($"a", $"b", $"c"),
--        regexp_extract($"a", "(\\d+)-(\\d+)", 1)),
--      Row("num-num", "300", "100") :: Row("num-num", "400", "100") ::
--        Row("num-num", "400-400", "100") :: Nil)
--
--    // for testing the mutable state of the expression in code gen.
--    // This is a hack way to enable the codegen, thus the codegen is enable by default,
--    // it will still use the interpretProjection if projection followed by a LocalRelation,
--    // hence we add a filter operator.
--    // See the optimizer rule `ConvertToLocalRelation`
--    checkAnswer(
--      df.filter("isnotnull(a)").selectExpr(
--        "regexp_replace(a, b, c)",
--        "regexp_extract(a, b, 1)"),
--      Row("300", "100") :: Row("400", "100") :: Row("400-400", "100") :: Nil)
-+      checkAnswer(
-+        df.select(
-+          regexp_replace($"a", "(\\d+)", "num"),
-+          regexp_replace($"a", $"b", $"c"),
-+          regexp_extract($"a", "(\\d+)-(\\d+)", 1)),
-+        Row("num-num", "300", "100") :: Row("num-num", "400", "100") ::
-+          Row("num-num", "400-400", "100") :: Nil)
-+
-+      // for testing the mutable state of the expression in code gen.
-+      // This is a hack way to enable the codegen, thus the codegen is enable by default,
-+      // it will still use the interpretProjection if projection followed by a LocalRelation,
-+      // hence we add a filter operator.
-+      // See the optimizer rule `ConvertToLocalRelation`
-+      checkAnswer(
-+        df.filter("isnotnull(a)").selectExpr(
-+          "regexp_replace(a, b, c)",
-+          "regexp_extract(a, b, 1)"),
-+        Row("300", "100") :: Row("400", "100") :: Row("400-400", "100") :: Nil)
-+    }
-   }
- 
-   test("non-matching optional group") {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-index 04702201f82..5ee11f83ecf 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-@@ -22,10 +22,11 @@ import scala.collection.mutable.ArrayBuffer
- import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
- import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi}
- import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Join, LogicalPlan, Project, Sort, Union}
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
- import org.apache.spark.sql.execution.datasources.FileScanRDD
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, BroadcastNestedLoopJoinExec}
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
-@@ -1599,6 +1600,12 @@ class SubquerySuite extends QueryTest
-             fs.inputRDDs().forall(
-               _.asInstanceOf[FileScanRDD].filePartitions.forall(
-                 _.files.forall(_.urlEncodedPath.contains("p=0"))))
-+        case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(
-+        fs @ CometScanExec(_, _, _, _, partitionFilters, _, _, _, _, _, _)))) =>
-+          partitionFilters.exists(ExecSubqueryExpression.hasSubquery) &&
-+            fs.inputRDDs().forall(
-+              _.asInstanceOf[FileScanRDD].filePartitions.forall(
-+                _.files.forall(_.urlEncodedPath.contains("p=0"))))
-         case _ => false
-       })
-     }
-@@ -2164,7 +2171,7 @@ class SubquerySuite extends QueryTest
- 
-       df.collect()
-       val exchanges = collect(df.queryExecution.executedPlan) {
--        case s: ShuffleExchangeExec => s
-+        case s: ShuffleExchangeLike => s
-       }
-       assert(exchanges.size === 1)
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
-index d269290e616..13726a31e07 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
-@@ -24,6 +24,7 @@ import test.org.apache.spark.sql.connector._
- 
- import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
- import org.apache.spark.sql.catalyst.InternalRow
-+import org.apache.spark.sql.comet.CometSortExec
- import org.apache.spark.sql.connector.catalog.{PartitionInternalRow, SupportsRead, Table, TableCapability, TableProvider}
- import org.apache.spark.sql.connector.catalog.TableCapability._
- import org.apache.spark.sql.connector.expressions.{Expression, FieldReference, Literal, NamedReference, NullOrdering, SortDirection, SortOrder, Transform}
-@@ -34,7 +35,7 @@ import org.apache.spark.sql.connector.read.partitioning.{KeyGroupedPartitioning,
- import org.apache.spark.sql.execution.SortExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV2Relation, DataSourceV2ScanRelation}
--import org.apache.spark.sql.execution.exchange.{Exchange, ShuffleExchangeExec}
-+import org.apache.spark.sql.execution.exchange.{Exchange, ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
- import org.apache.spark.sql.expressions.Window
- import org.apache.spark.sql.functions._
-@@ -269,13 +270,13 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
-           val groupByColJ = df.groupBy($"j").agg(sum($"i"))
-           checkAnswer(groupByColJ, Seq(Row(2, 8), Row(4, 2), Row(6, 5)))
-           assert(collectFirst(groupByColJ.queryExecution.executedPlan) {
--            case e: ShuffleExchangeExec => e
-+            case e: ShuffleExchangeLike => e
-           }.isDefined)
- 
-           val groupByIPlusJ = df.groupBy($"i" + $"j").agg(count("*"))
-           checkAnswer(groupByIPlusJ, Seq(Row(5, 2), Row(6, 2), Row(8, 1), Row(9, 1)))
-           assert(collectFirst(groupByIPlusJ.queryExecution.executedPlan) {
--            case e: ShuffleExchangeExec => e
-+            case e: ShuffleExchangeLike => e
-           }.isDefined)
-         }
-       }
-@@ -335,10 +336,11 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
- 
-                 val (shuffleExpected, sortExpected) = groupByExpects
-                 assert(collectFirst(groupBy.queryExecution.executedPlan) {
--                  case e: ShuffleExchangeExec => e
-+                  case e: ShuffleExchangeLike => e
-                 }.isDefined === shuffleExpected)
-                 assert(collectFirst(groupBy.queryExecution.executedPlan) {
-                   case e: SortExec => e
-+                  case c: CometSortExec => c
-                 }.isDefined === sortExpected)
-               }
- 
-@@ -353,10 +355,11 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
- 
-                 val (shuffleExpected, sortExpected) = windowFuncExpects
-                 assert(collectFirst(windowPartByColIOrderByColJ.queryExecution.executedPlan) {
--                  case e: ShuffleExchangeExec => e
-+                  case e: ShuffleExchangeLike => e
-                 }.isDefined === shuffleExpected)
-                 assert(collectFirst(windowPartByColIOrderByColJ.queryExecution.executedPlan) {
-                   case e: SortExec => e
-+                  case c: CometSortExec => c
-                 }.isDefined === sortExpected)
-               }
-             }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-index cfc8b2cc845..c6fcfd7bd08 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-@@ -21,6 +21,7 @@ import scala.collection.mutable.ArrayBuffer
- import org.apache.spark.SparkConf
- import org.apache.spark.sql.{AnalysisException, QueryTest}
- import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability}
- import org.apache.spark.sql.connector.read.ScanBuilder
- import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
-@@ -184,7 +185,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest with SharedSparkSession {
-             val df = spark.read.format(format).load(path.getCanonicalPath)
-             checkAnswer(df, inputData.toDF())
-             assert(
--              df.queryExecution.executedPlan.exists(_.isInstanceOf[FileSourceScanExec]))
-+              df.queryExecution.executedPlan.exists {
-+                case _: FileSourceScanExec | _: CometScanExec => true
-+                case _ => false
-+              }
-+            )
-           }
-         } finally {
-           spark.listenerManager.unregister(listener)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-index 71e030f535e..d5ae6cbf3d5 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-@@ -22,6 +22,7 @@ import org.apache.spark.sql.{DataFrame, Row}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{Literal, TransformExpression}
- import org.apache.spark.sql.catalyst.plans.physical
-+import org.apache.spark.sql.comet.CometSortMergeJoinExec
- import org.apache.spark.sql.connector.catalog.Identifier
- import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog
- import org.apache.spark.sql.connector.catalog.functions._
-@@ -31,7 +32,7 @@ import org.apache.spark.sql.connector.expressions.Expressions._
- import org.apache.spark.sql.execution.SparkPlan
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.joins.SortMergeJoinExec
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.internal.SQLConf._
-@@ -282,13 +283,14 @@ class KeyGroupedPartitioningSuite extends DistributionAndOrderingSuiteBase {
-         Row("bbb", 20, 250.0), Row("bbb", 20, 350.0), Row("ccc", 30, 400.50)))
-   }
- 
--  private def collectShuffles(plan: SparkPlan): Seq[ShuffleExchangeExec] = {
-+  private def collectShuffles(plan: SparkPlan): Seq[ShuffleExchangeLike] = {
-     // here we skip collecting shuffle operators that are not associated with SMJ
-     collect(plan) {
-       case s: SortMergeJoinExec => s
-+      case c: CometSortMergeJoinExec => c.originalPlan
-     }.flatMap(smj =>
-       collect(smj) {
--        case s: ShuffleExchangeExec => s
-+        case s: ShuffleExchangeLike => s
-       })
-   }
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
-index 12007cd94cd..07020f201fb 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
-@@ -21,7 +21,7 @@ package org.apache.spark.sql.connector
- import java.sql.Date
- import java.util.Collections
- 
--import org.apache.spark.sql.{catalyst, AnalysisException, DataFrame, Row}
-+import org.apache.spark.sql.{catalyst, AnalysisException, DataFrame, IgnoreCometSuite, Row}
- import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression, Cast, Literal}
- import org.apache.spark.sql.catalyst.expressions.objects.Invoke
- import org.apache.spark.sql.catalyst.plans.physical
-@@ -45,7 +45,8 @@ import org.apache.spark.sql.util.QueryExecutionListener
- import org.apache.spark.tags.SlowSQLTest
- 
- @SlowSQLTest
--class WriteDistributionAndOrderingSuite extends DistributionAndOrderingSuiteBase {
-+class WriteDistributionAndOrderingSuite extends DistributionAndOrderingSuiteBase
-+  with IgnoreCometSuite {
-   import testImplicits._
- 
-   before {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
-index ae1c0a86a14..1d3b914fd64 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
-@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.permission.FsPermission
- import org.mockito.Mockito.{mock, spy, when}
- 
- import org.apache.spark._
--import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, QueryTest, Row, SaveMode}
-+import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, IgnoreComet, QueryTest, Row, SaveMode}
- import org.apache.spark.sql.catalyst.FunctionIdentifier
- import org.apache.spark.sql.catalyst.analysis.{NamedParameter, UnresolvedGenerator}
- import org.apache.spark.sql.catalyst.expressions.{Grouping, Literal, RowNumber}
-@@ -256,7 +256,8 @@ class QueryExecutionErrorsSuite
-   }
- 
-   test("INCONSISTENT_BEHAVIOR_CROSS_VERSION: " +
--    "compatibility with Spark 2.4/3.2 in reading/writing dates") {
-+    "compatibility with Spark 2.4/3.2 in reading/writing dates",
-+    IgnoreComet("Comet doesn't completely support datetime rebase mode yet")) {
- 
-     // Fail to read ancient datetime values.
-     withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_READ.key -> EXCEPTION.toString) {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-index 418ca3430bb..eb8267192f8 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-@@ -23,7 +23,7 @@ import scala.util.Random
- import org.apache.hadoop.fs.Path
- 
- import org.apache.spark.SparkConf
--import org.apache.spark.sql.{DataFrame, QueryTest}
-+import org.apache.spark.sql.{DataFrame, IgnoreComet, QueryTest}
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
- import org.apache.spark.sql.internal.SQLConf
-@@ -195,7 +195,7 @@ class DataSourceV2ScanExecRedactionSuite extends DataSourceScanRedactionTest {
-     }
-   }
- 
--  test("FileScan description") {
-+  test("FileScan description", IgnoreComet("Comet doesn't use BatchScan")) {
-     Seq("json", "orc", "parquet").foreach { format =>
-       withTempPath { path =>
-         val dir = path.getCanonicalPath
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
-index 743ec41dbe7..9f30d6c8e04 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
-@@ -53,6 +53,10 @@ class LogicalPlanTagInSparkPlanSuite extends TPCDSQuerySuite with DisableAdaptiv
-     case ColumnarToRowExec(i: InputAdapter) => isScanPlanTree(i.child)
-     case p: ProjectExec => isScanPlanTree(p.child)
-     case f: FilterExec => isScanPlanTree(f.child)
-+    // Comet produces scan plan tree like:
-+    // ColumnarToRow
-+    //  +- ReusedExchange
-+    case _: ReusedExchangeExec => false
-     case _: LeafExecNode => true
-     case _ => false
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
-index de24b8c82b0..1f835481290 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.rdd.RDD
--import org.apache.spark.sql.{execution, DataFrame, Row}
-+import org.apache.spark.sql.{execution, DataFrame, IgnoreCometSuite, Row}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions._
- import org.apache.spark.sql.catalyst.plans._
-@@ -35,7 +35,9 @@ import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.sql.types._
- 
--class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
-+// Ignore this suite when Comet is enabled. This suite tests the Spark planner and Comet planner
-+// comes out with too many difference. Simply ignoring this suite for now.
-+class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper with IgnoreCometSuite {
-   import testImplicits._
- 
-   setupTestData()
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
-index 9e9d717db3b..91a4f9a38d5 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.sql.{DataFrame, QueryTest, Row}
-+import org.apache.spark.sql.comet.CometProjectExec
- import org.apache.spark.sql.connector.SimpleWritableDataSource
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.internal.SQLConf
-@@ -34,7 +35,10 @@ abstract class RemoveRedundantProjectsSuiteBase
-   private def assertProjectExecCount(df: DataFrame, expected: Int): Unit = {
-     withClue(df.queryExecution) {
-       val plan = df.queryExecution.executedPlan
--      val actual = collectWithSubqueries(plan) { case p: ProjectExec => p }.size
-+      val actual = collectWithSubqueries(plan) {
-+        case p: ProjectExec => p
-+        case p: CometProjectExec => p
-+      }.size
-       assert(actual == expected)
-     }
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
-index 005e764cc30..92ec088efab 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
-@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
- 
- import org.apache.spark.sql.{DataFrame, QueryTest}
- import org.apache.spark.sql.catalyst.plans.physical.{RangePartitioning, UnknownPartitioning}
-+import org.apache.spark.sql.comet.CometSortExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.execution.joins.ShuffledJoin
- import org.apache.spark.sql.internal.SQLConf
-@@ -33,7 +34,7 @@ abstract class RemoveRedundantSortsSuiteBase
- 
-   private def checkNumSorts(df: DataFrame, count: Int): Unit = {
-     val plan = df.queryExecution.executedPlan
--    assert(collectWithSubqueries(plan) { case s: SortExec => s }.length == count)
-+    assert(collectWithSubqueries(plan) { case _: SortExec | _: CometSortExec => 1 }.length == count)
-   }
- 
-   private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
-index 47679ed7865..9ffbaecb98e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.sql.{DataFrame, QueryTest}
-+import org.apache.spark.sql.comet.CometHashAggregateExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
- import org.apache.spark.sql.internal.SQLConf
-@@ -31,7 +32,7 @@ abstract class ReplaceHashWithSortAggSuiteBase
-   private def checkNumAggs(df: DataFrame, hashAggCount: Int, sortAggCount: Int): Unit = {
-     val plan = df.queryExecution.executedPlan
-     assert(collectWithSubqueries(plan) {
--      case s @ (_: HashAggregateExec | _: ObjectHashAggregateExec) => s
-+      case s @ (_: HashAggregateExec | _: ObjectHashAggregateExec | _: CometHashAggregateExec ) => s
-     }.length == hashAggCount)
-     assert(collectWithSubqueries(plan) { case s: SortAggregateExec => s }.length == sortAggCount)
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
-index eec396b2e39..bf3f1c769d6 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.execution
- 
- import org.apache.spark.TestUtils.assertSpilled
--import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, QueryTest, Row}
- import org.apache.spark.sql.internal.SQLConf.{WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD, WINDOW_EXEC_BUFFER_SPILL_THRESHOLD}
- import org.apache.spark.sql.test.SharedSparkSession
- 
-@@ -470,7 +470,7 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
-       Row(1, 3, null) :: Row(2, null, 4) :: Nil)
-   }
- 
--  test("test with low buffer spill threshold") {
-+  test("test with low buffer spill threshold", IgnoreComet("Comet does not support spilling")) {
-     val nums = sparkContext.parallelize(1 to 10).map(x => (x, x % 2)).toDF("x", "y")
-     nums.createOrReplaceTempView("nums")
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
-index b14f4a405f6..ab7baf434a5 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
-@@ -23,6 +23,7 @@ import org.apache.spark.sql.QueryTest
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
- import org.apache.spark.sql.catalyst.plans.logical.Deduplicate
-+import org.apache.spark.sql.comet.CometColumnarToRowExec
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
-@@ -131,7 +132,10 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession {
-         spark.range(1).write.parquet(path.getAbsolutePath)
-         val df = spark.read.parquet(path.getAbsolutePath)
-         val columnarToRowExec =
--          df.queryExecution.executedPlan.collectFirst { case p: ColumnarToRowExec => p }.get
-+          df.queryExecution.executedPlan.collectFirst {
-+            case p: ColumnarToRowExec => p
-+            case p: CometColumnarToRowExec => p
-+          }.get
-         try {
-           spark.range(1).foreach { _ =>
-             columnarToRowExec.canonicalized
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
-index 5a413c77754..207b66e1d7b 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
-@@ -17,7 +17,7 @@
- 
- package org.apache.spark.sql.execution
- 
--import org.apache.spark.sql.{Dataset, QueryTest, Row, SaveMode}
-+import org.apache.spark.sql.{Dataset, IgnoreCometSuite, QueryTest, Row, SaveMode}
- import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
- import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeAndComment, CodeGenerator}
- import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
-@@ -30,7 +30,7 @@ import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
- 
- // Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
--class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
-+class WholeStageCodegenSuite extends QueryTest with SharedSparkSession with IgnoreCometSuite
-   with DisableAdaptiveExecutionSuite {
- 
-   import testImplicits._
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
-index 2f8e401e743..a4f94417dcc 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
-@@ -27,9 +27,11 @@ import org.scalatest.time.SpanSugar._
- import org.apache.spark.SparkException
- import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
- import org.apache.spark.shuffle.sort.SortShuffleManager
--import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession, Strategy}
-+import org.apache.spark.sql.{Dataset, IgnoreComet, QueryTest, Row, SparkSession, Strategy}
- import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
- import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
-+import org.apache.spark.sql.comet._
-+import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.aggregate.BaseAggregateExec
- import org.apache.spark.sql.execution.columnar.{InMemoryTableScanExec, InMemoryTableScanLike}
-@@ -117,6 +119,7 @@ class AdaptiveQueryExecSuite
-   private def findTopLevelBroadcastHashJoin(plan: SparkPlan): Seq[BroadcastHashJoinExec] = {
-     collect(plan) {
-       case j: BroadcastHashJoinExec => j
-+      case j: CometBroadcastHashJoinExec => j.originalPlan.asInstanceOf[BroadcastHashJoinExec]
-     }
-   }
- 
-@@ -129,36 +132,46 @@ class AdaptiveQueryExecSuite
-   private def findTopLevelSortMergeJoin(plan: SparkPlan): Seq[SortMergeJoinExec] = {
-     collect(plan) {
-       case j: SortMergeJoinExec => j
-+      case j: CometSortMergeJoinExec =>
-+        assert(j.originalPlan.isInstanceOf[SortMergeJoinExec])
-+        j.originalPlan.asInstanceOf[SortMergeJoinExec]
-     }
-   }
- 
-   private def findTopLevelShuffledHashJoin(plan: SparkPlan): Seq[ShuffledHashJoinExec] = {
-     collect(plan) {
-       case j: ShuffledHashJoinExec => j
-+      case j: CometHashJoinExec => j.originalPlan.asInstanceOf[ShuffledHashJoinExec]
-     }
-   }
- 
-   private def findTopLevelBaseJoin(plan: SparkPlan): Seq[BaseJoinExec] = {
-     collect(plan) {
-       case j: BaseJoinExec => j
-+      case c: CometHashJoinExec => c.originalPlan.asInstanceOf[BaseJoinExec]
-+      case c: CometSortMergeJoinExec => c.originalPlan.asInstanceOf[BaseJoinExec]
-+      case c: CometBroadcastHashJoinExec => c.originalPlan.asInstanceOf[BaseJoinExec]
-     }
-   }
- 
-   private def findTopLevelSort(plan: SparkPlan): Seq[SortExec] = {
-     collect(plan) {
-       case s: SortExec => s
-+      case s: CometSortExec => s.originalPlan.asInstanceOf[SortExec]
-     }
-   }
- 
-   private def findTopLevelAggregate(plan: SparkPlan): Seq[BaseAggregateExec] = {
-     collect(plan) {
-       case agg: BaseAggregateExec => agg
-+      case agg: CometHashAggregateExec => agg.originalPlan.asInstanceOf[BaseAggregateExec]
-     }
-   }
- 
-   private def findTopLevelLimit(plan: SparkPlan): Seq[CollectLimitExec] = {
-     collect(plan) {
-       case l: CollectLimitExec => l
-+      case l: CometCollectLimitExec => l.originalPlan.asInstanceOf[CollectLimitExec]
-     }
-   }
- 
-@@ -202,6 +215,7 @@ class AdaptiveQueryExecSuite
-       val parts = rdd.partitions
-       assert(parts.forall(rdd.preferredLocations(_).nonEmpty))
-     }
-+
-     assert(numShuffles === (numLocalReads.length + numShufflesWithoutLocalRead))
-   }
- 
-@@ -210,7 +224,7 @@ class AdaptiveQueryExecSuite
-     val plan = df.queryExecution.executedPlan
-     assert(plan.isInstanceOf[AdaptiveSparkPlanExec])
-     val shuffle = plan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan.collect {
--      case s: ShuffleExchangeExec => s
-+      case s: ShuffleExchangeLike => s
-     }
-     assert(shuffle.size == 1)
-     assert(shuffle(0).outputPartitioning.numPartitions == numPartition)
-@@ -226,7 +240,8 @@ class AdaptiveQueryExecSuite
-       assert(smj.size == 1)
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
--      checkNumLocalShuffleReads(adaptivePlan)
-+      // Comet shuffle changes shuffle metrics
-+      // checkNumLocalShuffleReads(adaptivePlan)
-     }
-   }
- 
-@@ -253,7 +268,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Reuse the parallelism of coalesced shuffle in local shuffle read") {
-+  test("Reuse the parallelism of coalesced shuffle in local shuffle read",
-+      IgnoreComet("Comet shuffle changes shuffle partition size")) {
-     withSQLConf(
-       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-@@ -285,7 +301,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Reuse the default parallelism in local shuffle read") {
-+  test("Reuse the default parallelism in local shuffle read",
-+      IgnoreComet("Comet shuffle changes shuffle partition size")) {
-     withSQLConf(
-       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-@@ -299,7 +316,8 @@ class AdaptiveQueryExecSuite
-       val localReads = collect(adaptivePlan) {
-         case read: AQEShuffleReadExec if read.isLocalRead => read
-       }
--      assert(localReads.length == 2)
-+      // Comet shuffle changes shuffle metrics
-+      assert(localReads.length == 1)
-       val localShuffleRDD0 = localReads(0).execute().asInstanceOf[ShuffledRowRDD]
-       val localShuffleRDD1 = localReads(1).execute().asInstanceOf[ShuffledRowRDD]
-       // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2
-@@ -324,7 +342,9 @@ class AdaptiveQueryExecSuite
-           .groupBy($"a").count()
-         checkAnswer(testDf, Seq())
-         val plan = testDf.queryExecution.executedPlan
--        assert(find(plan)(_.isInstanceOf[SortMergeJoinExec]).isDefined)
-+        assert(find(plan) { case p =>
-+          p.isInstanceOf[SortMergeJoinExec] || p.isInstanceOf[CometSortMergeJoinExec]
-+        }.isDefined)
-         val coalescedReads = collect(plan) {
-           case r: AQEShuffleReadExec => r
-         }
-@@ -338,7 +358,9 @@ class AdaptiveQueryExecSuite
-           .groupBy($"a").count()
-         checkAnswer(testDf, Seq())
-         val plan = testDf.queryExecution.executedPlan
--        assert(find(plan)(_.isInstanceOf[BroadcastHashJoinExec]).isDefined)
-+        assert(find(plan) { case p =>
-+          p.isInstanceOf[BroadcastHashJoinExec] || p.isInstanceOf[CometBroadcastHashJoinExec]
-+        }.isDefined)
-         val coalescedReads = collect(plan) {
-           case r: AQEShuffleReadExec => r
-         }
-@@ -348,7 +370,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Scalar subquery") {
-+  test("Scalar subquery", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -363,7 +385,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Scalar subquery in later stages") {
-+  test("Scalar subquery in later stages", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -379,7 +401,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("multiple joins") {
-+  test("multiple joins", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -424,7 +446,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("multiple joins with aggregate") {
-+  test("multiple joins with aggregate", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -469,7 +491,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("multiple joins with aggregate 2") {
-+  test("multiple joins with aggregate 2", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") {
-@@ -515,7 +537,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Exchange reuse") {
-+  test("Exchange reuse", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -534,7 +556,7 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Exchange reuse with subqueries") {
-+  test("Exchange reuse with subqueries", IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
-@@ -565,7 +587,9 @@ class AdaptiveQueryExecSuite
-       assert(smj.size == 1)
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
--      checkNumLocalShuffleReads(adaptivePlan)
-+      // Comet shuffle changes shuffle metrics,
-+      // so we can't check the number of local shuffle reads.
-+      // checkNumLocalShuffleReads(adaptivePlan)
-       // Even with local shuffle read, the query stage reuse can also work.
-       val ex = findReusedExchange(adaptivePlan)
-       assert(ex.nonEmpty)
-@@ -586,7 +610,9 @@ class AdaptiveQueryExecSuite
-       assert(smj.size == 1)
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
--      checkNumLocalShuffleReads(adaptivePlan)
-+      // Comet shuffle changes shuffle metrics,
-+      // so we can't check the number of local shuffle reads.
-+      // checkNumLocalShuffleReads(adaptivePlan)
-       // Even with local shuffle read, the query stage reuse can also work.
-       val ex = findReusedExchange(adaptivePlan)
-       assert(ex.isEmpty)
-@@ -595,7 +621,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("Broadcast exchange reuse across subqueries") {
-+  test("Broadcast exchange reuse across subqueries",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "20000000",
-@@ -690,7 +717,8 @@ class AdaptiveQueryExecSuite
-       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
-       assert(bhj.size == 1)
-       // There is still a SMJ, and its two shuffles can't apply local read.
--      checkNumLocalShuffleReads(adaptivePlan, 2)
-+      // Comet shuffle changes shuffle metrics
-+      // checkNumLocalShuffleReads(adaptivePlan, 2)
-     }
-   }
- 
-@@ -812,7 +840,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-29544: adaptive skew join with different join types") {
-+  test("SPARK-29544: adaptive skew join with different join types",
-+      IgnoreComet("Comet shuffle has different partition metrics")) {
-     Seq("SHUFFLE_MERGE", "SHUFFLE_HASH").foreach { joinHint =>
-       def getJoinNode(plan: SparkPlan): Seq[ShuffledJoin] = if (joinHint == "SHUFFLE_MERGE") {
-         findTopLevelSortMergeJoin(plan)
-@@ -1030,7 +1059,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("metrics of the shuffle read") {
-+  test("metrics of the shuffle read",
-+      IgnoreComet("Comet shuffle changes the metrics")) {
-     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
-       val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
-         "SELECT key FROM testData GROUP BY key")
-@@ -1625,7 +1655,7 @@ class AdaptiveQueryExecSuite
-         val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
-           "SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id")
-         assert(collect(adaptivePlan) {
--          case s: ShuffleExchangeExec => s
-+          case s: ShuffleExchangeLike => s
-         }.length == 1)
-       }
-     }
-@@ -1705,7 +1735,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-33551: Do not use AQE shuffle read for repartition") {
-+  test("SPARK-33551: Do not use AQE shuffle read for repartition",
-+      IgnoreComet("Comet shuffle changes partition size")) {
-     def hasRepartitionShuffle(plan: SparkPlan): Boolean = {
-       find(plan) {
-         case s: ShuffleExchangeLike =>
-@@ -1890,6 +1921,9 @@ class AdaptiveQueryExecSuite
-     def checkNoCoalescePartitions(ds: Dataset[Row], origin: ShuffleOrigin): Unit = {
-       assert(collect(ds.queryExecution.executedPlan) {
-         case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s
-+        case c: CometShuffleExchangeExec
-+          if c.originalPlan.shuffleOrigin == origin &&
-+            c.originalPlan.numPartitions == 2 => c
-       }.size == 1)
-       ds.collect()
-       val plan = ds.queryExecution.executedPlan
-@@ -1898,6 +1932,9 @@ class AdaptiveQueryExecSuite
-       }.isEmpty)
-       assert(collect(plan) {
-         case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s
-+        case c: CometShuffleExchangeExec
-+          if c.originalPlan.shuffleOrigin == origin &&
-+            c.originalPlan.numPartitions == 2 => c
-       }.size == 1)
-       checkAnswer(ds, testData)
-     }
-@@ -2054,7 +2091,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-35264: Support AQE side shuffled hash join formula") {
-+  test("SPARK-35264: Support AQE side shuffled hash join formula",
-+      IgnoreComet("Comet shuffle changes the partition size")) {
-     withTempView("t1", "t2") {
-       def checkJoinStrategy(shouldShuffleHashJoin: Boolean): Unit = {
-         Seq("100", "100000").foreach { size =>
-@@ -2140,7 +2178,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-35725: Support optimize skewed partitions in RebalancePartitions") {
-+  test("SPARK-35725: Support optimize skewed partitions in RebalancePartitions",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withTempView("v") {
-       withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-@@ -2239,7 +2278,7 @@ class AdaptiveQueryExecSuite
-               runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM skewData1 " +
-                 s"JOIN skewData2 ON key1 = key2 GROUP BY key1")
-             val shuffles1 = collect(adaptive1) {
--              case s: ShuffleExchangeExec => s
-+              case s: ShuffleExchangeLike => s
-             }
-             assert(shuffles1.size == 3)
-             // shuffles1.head is the top-level shuffle under the Aggregate operator
-@@ -2252,7 +2291,7 @@ class AdaptiveQueryExecSuite
-               runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM skewData1 " +
-                 s"JOIN skewData2 ON key1 = key2")
-             val shuffles2 = collect(adaptive2) {
--              case s: ShuffleExchangeExec => s
-+              case s: ShuffleExchangeLike => s
-             }
-             if (hasRequiredDistribution) {
-               assert(shuffles2.size == 3)
-@@ -2286,7 +2325,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-35794: Allow custom plugin for cost evaluator") {
-+  test("SPARK-35794: Allow custom plugin for cost evaluator",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     CostEvaluator.instantiate(
-       classOf[SimpleShuffleSortCostEvaluator].getCanonicalName, spark.sparkContext.getConf)
-     intercept[IllegalArgumentException] {
-@@ -2417,7 +2457,8 @@ class AdaptiveQueryExecSuite
-   }
- 
-   test("SPARK-48037: Fix SortShuffleWriter lacks shuffle write related metrics " +
--    "resulting in potentially inaccurate data") {
-+    "resulting in potentially inaccurate data",
-+    IgnoreComet("https://github.com/apache/datafusion-comet/issues/1501")) {
-     withTable("t3") {
-       withSQLConf(
-         SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-@@ -2452,6 +2493,7 @@ class AdaptiveQueryExecSuite
-           val (_, adaptive) = runAdaptiveAndVerifyResult(query)
-           assert(adaptive.collect {
-             case sort: SortExec => sort
-+            case sort: CometSortExec => sort
-           }.size == 1)
-           val read = collect(adaptive) {
-             case read: AQEShuffleReadExec => read
-@@ -2469,7 +2511,8 @@ class AdaptiveQueryExecSuite
-     }
-   }
- 
--  test("SPARK-37357: Add small partition factor for rebalance partitions") {
-+  test("SPARK-37357: Add small partition factor for rebalance partitions",
-+      IgnoreComet("Comet shuffle changes shuffle metrics")) {
-     withTempView("v") {
-       withSQLConf(
-         SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED.key -> "true",
-@@ -2581,7 +2624,7 @@ class AdaptiveQueryExecSuite
-           runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
-             "JOIN skewData3 ON value2 = value3")
-         val shuffles1 = collect(adaptive1) {
--          case s: ShuffleExchangeExec => s
-+          case s: ShuffleExchangeLike => s
-         }
-         assert(shuffles1.size == 4)
-         val smj1 = findTopLevelSortMergeJoin(adaptive1)
-@@ -2592,7 +2635,7 @@ class AdaptiveQueryExecSuite
-           runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
-             "JOIN skewData3 ON value1 = value3")
-         val shuffles2 = collect(adaptive2) {
--          case s: ShuffleExchangeExec => s
-+          case s: ShuffleExchangeLike => s
-         }
-         assert(shuffles2.size == 4)
-         val smj2 = findTopLevelSortMergeJoin(adaptive2)
-@@ -2850,6 +2893,7 @@ class AdaptiveQueryExecSuite
-         }.size == (if (firstAccess) 1 else 0))
-         assert(collect(initialExecutedPlan) {
-           case s: SortExec => s
-+          case s: CometSortExec => s
-         }.size == (if (firstAccess) 2 else 0))
-         assert(collect(initialExecutedPlan) {
-           case i: InMemoryTableScanLike => i
-@@ -2980,7 +3024,9 @@ class AdaptiveQueryExecSuite
- 
-       val plan = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec]
-       assert(plan.inputPlan.isInstanceOf[TakeOrderedAndProjectExec])
--      assert(plan.finalPhysicalPlan.isInstanceOf[WindowExec])
-+      assert(
-+        plan.finalPhysicalPlan.isInstanceOf[WindowExec] ||
-+        plan.finalPhysicalPlan.find(_.isInstanceOf[CometWindowExec]).nonEmpty)
-       plan.inputPlan.output.zip(plan.finalPhysicalPlan.output).foreach { case (o1, o2) =>
-         assert(o1.semanticEquals(o2), "Different output column order after AQE optimization")
-       }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-index 05872d41131..a2c328b9742 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-@@ -21,7 +21,7 @@ import java.io.File
- 
- import org.apache.hadoop.fs.{FileStatus, Path}
- 
--import org.apache.spark.sql.{DataFrame, Dataset, QueryTest, Row}
-+import org.apache.spark.sql.{DataFrame, Dataset, IgnoreComet, QueryTest, Row}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{Expression, FileSourceConstantMetadataStructField, FileSourceGeneratedMetadataStructField, Literal}
- import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
-@@ -134,7 +134,8 @@ class FileSourceCustomMetadataStructSuite extends QueryTest with SharedSparkSess
-     }
-   }
- 
--  test("[SPARK-43226] extra constant metadata fields with extractors") {
-+  test("[SPARK-43226] extra constant metadata fields with extractors",
-+      IgnoreComet("TODO: fix Comet for this test")) {
-     withTempData("parquet", FILE_SCHEMA) { (_, f0, f1) =>
-       val format = new TestFileFormat(extraConstantMetadataFields) {
-         val extractPartitionNumber = { pf: PartitionedFile =>
-@@ -335,7 +336,8 @@ class FileSourceCustomMetadataStructSuite extends QueryTest with SharedSparkSess
-     }
-   }
- 
--  test("generated columns and extractors take precedence over metadata map values") {
-+  test("generated columns and extractors take precedence over metadata map values",
-+      IgnoreComet("TODO: fix Comet for this test")) {
-     withTempData("parquet", FILE_SCHEMA) { (_, f0, f1) =>
-       import FileFormat.{FILE_NAME, FILE_SIZE}
-       import ParquetFileFormat.ROW_INDEX
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
-index bf496d6db21..9bb57a9b4c6 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
-@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.Concat
- import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
- import org.apache.spark.sql.catalyst.plans.logical.Expand
- import org.apache.spark.sql.catalyst.types.DataTypeUtils
-+import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec}
- import org.apache.spark.sql.execution.FileSourceScanExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.functions._
-@@ -868,6 +869,8 @@ abstract class SchemaPruningSuite
-     val fileSourceScanSchemata =
-       collect(df.queryExecution.executedPlan) {
-         case scan: FileSourceScanExec => scan.requiredSchema
-+        case scan: CometScanExec => scan.requiredSchema
-+        case scan: CometNativeScanExec => scan.requiredSchema
-       }
-     assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
-       s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala
-index ce43edb79c1..4dbb5942bc3 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/V1WriteCommandSuite.scala
-@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources
- import org.apache.spark.sql.{QueryTest, Row}
- import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, NullsFirst, SortOrder}
- import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Sort}
-+import org.apache.spark.sql.comet.CometSortExec
- import org.apache.spark.sql.execution.{QueryExecution, SortExec}
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
- import org.apache.spark.sql.internal.SQLConf
-@@ -225,6 +226,7 @@ class V1WriteCommandSuite extends QueryTest with SharedSparkSession with V1Write
-           // assert the outer most sort in the executed plan
-           assert(plan.collectFirst {
-             case s: SortExec => s
-+            case s: CometSortExec => s.originalPlan.asInstanceOf[SortExec]
-           }.exists {
-             case SortExec(Seq(
-               SortOrder(AttributeReference("key", IntegerType, _, _), Ascending, NullsFirst, _),
-@@ -272,6 +274,7 @@ class V1WriteCommandSuite extends QueryTest with SharedSparkSession with V1Write
-         // assert the outer most sort in the executed plan
-         assert(plan.collectFirst {
-           case s: SortExec => s
-+          case s: CometSortExec => s.originalPlan.asInstanceOf[SortExec]
-         }.exists {
-           case SortExec(Seq(
-             SortOrder(AttributeReference("value", StringType, _, _), Ascending, NullsFirst, _),
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
-index 0b6fdef4f74..5b18c55da4b 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
-@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, GlobFilter, Path}
- import org.mockito.Mockito.{mock, when}
- 
- import org.apache.spark.SparkException
--import org.apache.spark.sql.{DataFrame, QueryTest, Row}
-+import org.apache.spark.sql.{DataFrame, IgnoreCometSuite, QueryTest, Row}
- import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
- import org.apache.spark.sql.execution.datasources.PartitionedFile
- import org.apache.spark.sql.functions.col
-@@ -38,7 +38,9 @@ import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.sql.types._
- import org.apache.spark.util.Utils
- 
--class BinaryFileFormatSuite extends QueryTest with SharedSparkSession {
-+// For some reason this suite is flaky w/ or w/o Comet when running in Github workflow.
-+// Since it isn't related to Comet, we disable it for now.
-+class BinaryFileFormatSuite extends QueryTest with SharedSparkSession with IgnoreCometSuite {
-   import BinaryFileFormat._
- 
-   private var testDir: String = _
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
-index 07e2849ce6f..3e73645b638 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
-@@ -28,7 +28,7 @@ import org.apache.parquet.hadoop.ParquetOutputFormat
- 
- import org.apache.spark.TestUtils
- import org.apache.spark.memory.MemoryMode
--import org.apache.spark.sql.Row
-+import org.apache.spark.sql.{IgnoreComet, Row}
- import org.apache.spark.sql.catalyst.util.DateTimeUtils
- import org.apache.spark.sql.internal.SQLConf
- import org.apache.spark.sql.test.SharedSparkSession
-@@ -201,7 +201,8 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
-     }
-   }
- 
--  test("parquet v2 pages - rle encoding for boolean value columns") {
-+  test("parquet v2 pages - rle encoding for boolean value columns",
-+      IgnoreComet("Comet doesn't support RLE encoding yet")) {
-     val extraOptions = Map[String, String](
-       ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
-     )
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 8e88049f51e..5a963dab6f2 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-@@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-           // When a filter is pushed to Parquet, Parquet can apply it to every row.
-           // So, we can check the number of rows returned from the Parquet
-           // to make sure our filter pushdown work.
--          assert(stripSparkFilter(df).count == 1)
-+          // Similar to Spark's vectorized reader, Comet doesn't do row-level filtering but relies
-+          // on Spark to apply the data filters after columnar batches are returned
-+          if (!isCometEnabled) {
-+            assert(stripSparkFilter(df).count == 1)
-+          }
-         }
-       }
-     }
-@@ -1498,7 +1502,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("Filters should be pushed down for vectorized Parquet reader at row group level") {
-+  test("Filters should be pushed down for vectorized Parquet reader at row group level",
-+    IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
-     import testImplicits._
- 
-     withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
-@@ -1580,7 +1585,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-           // than the total length but should not be a single record.
-           // Note that, if record level filtering is enabled, it should be a single record.
-           // If no filter is pushed down to Parquet, it should be the total length of data.
--          assert(actual > 1 && actual < data.length)
-+          // Only enable Comet test iff it's scan only, since with native execution
-+          // `stripSparkFilter` can't remove the native filter
-+          if (!isCometEnabled || isCometScanOnly) {
-+            assert(actual > 1 && actual < data.length)
-+          }
-         }
-       }
-     }
-@@ -1607,7 +1616,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-         // than the total length but should not be a single record.
-         // Note that, if record level filtering is enabled, it should be a single record.
-         // If no filter is pushed down to Parquet, it should be the total length of data.
--        assert(actual > 1 && actual < data.length)
-+        // Only enable Comet test iff it's scan only, since with native execution
-+        // `stripSparkFilter` can't remove the native filter
-+        if (!isCometEnabled || isCometScanOnly) {
-+          assert(actual > 1 && actual < data.length)
-+        }
-       }
-     }
-   }
-@@ -1699,7 +1712,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-       (attr, value) => sources.StringContains(attr, value))
-   }
- 
--  test("filter pushdown - StringPredicate") {
-+  test("filter pushdown - StringPredicate", IgnoreCometNativeScan("cannot be pushed down")) {
-     import testImplicits._
-     // keep() should take effect on StartsWith/EndsWith/Contains
-     Seq(
-@@ -1743,7 +1756,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
-+  test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+      IgnoreComet("IN predicate is not yet supported in Comet, see issue #36")) {
-     val schema = StructType(Seq(
-       StructField("a", IntegerType, nullable = false)
-     ))
-@@ -1984,7 +1998,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("Support Parquet column index") {
-+  test("Support Parquet column index",
-+      IgnoreComet("Comet doesn't support Parquet column index yet")) {
-     // block 1:
-     //                      null count  min                                       max
-     // page-0                         0  0                                         99
-@@ -2044,7 +2059,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("SPARK-34562: Bloom filter push down") {
-+  test("SPARK-34562: Bloom filter push down",
-+    IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
-     withTempPath { dir =>
-       val path = dir.getCanonicalPath
-       spark.range(100).selectExpr("id * 2 AS id")
-@@ -2276,7 +2292,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
-           assert(pushedParquetFilters.exists(_.getClass === filterClass),
-             s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
- 
--          checker(stripSparkFilter(query), expected)
-+          // Similar to Spark's vectorized reader, Comet doesn't do row-level filtering but relies
-+          // on Spark to apply the data filters after columnar batches are returned
-+          if (!isCometEnabled) {
-+            checker(stripSparkFilter(query), expected)
-+          }
-         } else {
-           assert(selectedFilters.isEmpty, "There is filter pushed down")
-         }
-@@ -2336,7 +2356,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
-           assert(pushedParquetFilters.exists(_.getClass === filterClass),
-             s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
- 
--          checker(stripSparkFilter(query), expected)
-+          // Similar to Spark's vectorized reader, Comet doesn't do row-level filtering but relies
-+          // on Spark to apply the data filters after columnar batches are returned
-+          if (!isCometEnabled) {
-+            checker(stripSparkFilter(query), expected)
-+          }
- 
-         case _ =>
-           throw new AnalysisException("Can not match ParquetTable in the query.")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-index 4f8a9e39716..fb55ac7a955 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-@@ -1335,7 +1335,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
-     }
-   }
- 
--  test("SPARK-40128 read DELTA_LENGTH_BYTE_ARRAY encoded strings") {
-+  test("SPARK-40128 read DELTA_LENGTH_BYTE_ARRAY encoded strings",
-+      IgnoreComet("Comet doesn't support DELTA encoding yet")) {
-     withAllParquetReaders {
-       checkAnswer(
-         // "fruit" column in this file is encoded using DELTA_LENGTH_BYTE_ARRAY.
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index f6472ba3d9d..7a8f5317ed7 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-@@ -998,7 +998,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-     }
-   }
- 
--  test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") {
-+  test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups",
-+    IgnoreCometNativeScan("Native scans had the filter pushed into DF operator, cannot strip")) {
-     withAllParquetReaders {
-       withTempPath { path =>
-         // Repeated values for dictionary encoding.
-@@ -1067,7 +1068,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-         checkAnswer(readParquet(schema, path), df)
-       }
- 
--      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
-+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
-+          "spark.comet.enabled" -> "false") {
-         val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
-         checkAnswer(readParquet(schema1, path), df)
-         val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
-@@ -1089,7 +1091,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-       val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c, CAST('1.2' AS BINARY) d")
-       df.write.parquet(path.toString)
- 
--      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
-+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
-+          "spark.comet.enabled" -> "false") {
-         checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00"))
-         checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null))
-         checkAnswer(readParquet("b DECIMAL(11, 1)", path), sql("SELECT 123456.0"))
-@@ -1148,7 +1151,7 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
-             .where(s"a < ${Long.MaxValue}")
-             .collect()
-         }
--        assert(exception.getCause.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
-+        assert(exception.getMessage.contains("Column: [a], Expected: bigint, Found: INT32"))
-       }
-     }
-   }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
-index 4f906411345..6cc69f7e915 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala
-@@ -21,7 +21,7 @@ import java.nio.file.{Files, Paths, StandardCopyOption}
- import java.sql.{Date, Timestamp}
- 
- import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException, SparkUpgradeException}
--import org.apache.spark.sql.{QueryTest, Row, SPARK_LEGACY_DATETIME_METADATA_KEY, SPARK_LEGACY_INT96_METADATA_KEY, SPARK_TIMEZONE_METADATA_KEY}
-+import org.apache.spark.sql.{IgnoreCometSuite, QueryTest, Row, SPARK_LEGACY_DATETIME_METADATA_KEY, SPARK_LEGACY_INT96_METADATA_KEY, SPARK_TIMEZONE_METADATA_KEY}
- import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
- import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}
- import org.apache.spark.sql.internal.LegacyBehaviorPolicy.{CORRECTED, EXCEPTION, LEGACY}
-@@ -30,9 +30,11 @@ import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType.{INT96,
- import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.tags.SlowSQLTest
- 
-+// Comet is disabled for this suite because it doesn't support datetime rebase mode
- abstract class ParquetRebaseDatetimeSuite
-   extends QueryTest
-   with ParquetTest
-+  with IgnoreCometSuite
-   with SharedSparkSession {
- 
-   import testImplicits._
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-index 27c2a2148fd..df04a15fb1f 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-@@ -20,12 +20,14 @@ import java.io.File
- 
- import scala.collection.JavaConverters._
- 
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.apache.parquet.column.ParquetProperties._
- import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
- import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
- 
- import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometScanExec}
- import org.apache.spark.sql.execution.FileSourceScanExec
- import org.apache.spark.sql.execution.datasources.FileFormat
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -172,6 +174,8 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
- 
-   private def testRowIndexGeneration(label: String, conf: RowIndexTestConf): Unit = {
-     test (s"$label - ${conf.desc}") {
-+      // native_datafusion Parquet scan does not support row index generation.
-+      assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_DATAFUSION)
-       withSQLConf(conf.sqlConfs: _*) {
-         withTempPath { path =>
-           // Read row index using _metadata.row_index if that is supported by the file format.
-@@ -243,6 +247,12 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
-             case f: FileSourceScanExec =>
-               numPartitions += f.inputRDD.partitions.length
-               numOutputRows += f.metrics("numOutputRows").value
-+            case b: CometScanExec =>
-+              numPartitions += b.inputRDD.partitions.length
-+              numOutputRows += b.metrics("numOutputRows").value
-+            case b: CometBatchScanExec =>
-+              numPartitions += b.inputRDD.partitions.length
-+              numOutputRows += b.metrics("numOutputRows").value
-             case _ =>
-           }
-           assert(numPartitions > 0)
-@@ -301,6 +311,8 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
-     val conf = RowIndexTestConf(useDataSourceV2 = useDataSourceV2)
- 
-     test(s"invalid row index column type - ${conf.desc}") {
-+      // native_datafusion Parquet scan does not support row index generation.
-+      assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_DATAFUSION)
-       withSQLConf(conf.sqlConfs: _*) {
-         withTempPath{ path =>
-           val df = spark.range(0, 10, 1, 1).toDF("id")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
-index 5c0b7def039..151184bc98c 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
-@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
- import org.apache.spark.SparkConf
- import org.apache.spark.sql.DataFrame
- import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-+import org.apache.spark.sql.comet.CometBatchScanExec
- import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
- import org.apache.spark.sql.execution.datasources.SchemaPruningSuite
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -56,6 +57,7 @@ class ParquetV2SchemaPruningSuite extends ParquetSchemaPruningSuite {
-     val fileSourceScanSchemata =
-       collect(df.queryExecution.executedPlan) {
-         case scan: BatchScanExec => scan.scan.asInstanceOf[ParquetScan].readDataSchema
-+        case scan: CometBatchScanExec => scan.scan.asInstanceOf[ParquetScan].readDataSchema
-       }
-     assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
-       s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-index 3f47c5e506f..bc1ee1ec0ba 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-@@ -27,6 +27,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
- import org.apache.parquet.schema.Type._
- 
- import org.apache.spark.SparkException
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.expressions.Cast.toSQLType
- import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
- import org.apache.spark.sql.functions.desc
-@@ -1036,7 +1037,8 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
-     e
-   }
- 
--  test("schema mismatch failure error message for parquet reader") {
-+  test("schema mismatch failure error message for parquet reader",
-+      IgnoreComet("Comet doesn't work with vectorizedReaderEnabled = false")) {
-     withTempPath { dir =>
-       val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = false)
-       val expectedMessage = "Encountered error while reading file"
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
-index b8f3ea3c6f3..bbd44221288 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
-@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.debug
- import java.io.ByteArrayOutputStream
- 
- import org.apache.spark.rdd.RDD
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.Attribute
- import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
-@@ -125,7 +126,8 @@ class DebuggingSuite extends DebuggingSuiteBase with DisableAdaptiveExecutionSui
-          | id LongType: {}""".stripMargin))
-   }
- 
--  test("SPARK-28537: DebugExec cannot debug columnar related queries") {
-+  test("SPARK-28537: DebugExec cannot debug columnar related queries",
-+      IgnoreComet("Comet does not use FileScan")) {
-     withTempPath { workDir =>
-       val workDirPath = workDir.getAbsolutePath
-       val input = spark.range(5).toDF("id")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
-index 5cdbdc27b32..307fba16578 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
-@@ -46,8 +46,10 @@ import org.apache.spark.sql.util.QueryExecutionListener
- import org.apache.spark.util.{AccumulatorContext, JsonProtocol}
- 
- // Disable AQE because metric info is different with AQE on/off
-+// This test suite runs tests against the metrics of physical operators.
-+// Disabling it for Comet because the metrics are different with Comet enabled.
- class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
--  with DisableAdaptiveExecutionSuite {
-+  with DisableAdaptiveExecutionSuite with IgnoreCometSuite {
-   import testImplicits._
- 
-   /**
-@@ -765,7 +767,8 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
-     }
-   }
- 
--  test("SPARK-26327: FileSourceScanExec metrics") {
-+  test("SPARK-26327: FileSourceScanExec metrics",
-+      IgnoreComet("Spark uses row-based Parquet reader while Comet is vectorized")) {
-     withTable("testDataForScan") {
-       spark.range(10).selectExpr("id", "id % 3 as p")
-         .write.partitionBy("p").saveAsTable("testDataForScan")
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
-index 0ab8691801d..d9125f658ad 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.execution.python
- 
- import org.apache.spark.sql.catalyst.plans.logical.{ArrowEvalPython, BatchEvalPython, Limit, LocalLimit}
-+import org.apache.spark.sql.comet._
- import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan, SparkPlanTest}
- import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
- import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
-@@ -108,6 +109,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: FileSourceScanExec => scan
-+            case scan: CometScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           assert(scanNodes.head.output.map(_.name) == Seq("a"))
-@@ -120,11 +122,16 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: FileSourceScanExec => scan
-+            case scan: CometScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           // $"a" is not null and $"a" > 1
--          assert(scanNodes.head.dataFilters.length == 2)
--          assert(scanNodes.head.dataFilters.flatMap(_.references.map(_.name)).distinct == Seq("a"))
-+          val dataFilters = scanNodes.head match {
-+            case scan: FileSourceScanExec => scan.dataFilters
-+            case scan: CometScanExec => scan.dataFilters
-+          }
-+          assert(dataFilters.length == 2)
-+          assert(dataFilters.flatMap(_.references.map(_.name)).distinct == Seq("a"))
-         }
-       }
-     }
-@@ -145,6 +152,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: BatchScanExec => scan
-+            case scan: CometBatchScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           assert(scanNodes.head.output.map(_.name) == Seq("a"))
-@@ -157,6 +165,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
- 
-           val scanNodes = query.queryExecution.executedPlan.collect {
-             case scan: BatchScanExec => scan
-+            case scan: CometBatchScanExec => scan
-           }
-           assert(scanNodes.length == 1)
-           // $"a" is not null and $"a" > 1
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala
-index d083cac48ff..3c11bcde807 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/AsyncProgressTrackingMicroBatchExecutionSuite.scala
-@@ -37,8 +37,10 @@ import org.apache.spark.sql.streaming.{StreamingQuery, StreamingQueryException,
- import org.apache.spark.sql.streaming.util.StreamManualClock
- import org.apache.spark.util.Utils
- 
-+// For some reason this suite is flaky w/ or w/o Comet when running in Github workflow.
-+// Since it isn't related to Comet, we disable it for now.
- class AsyncProgressTrackingMicroBatchExecutionSuite
--  extends StreamTest with BeforeAndAfter with Matchers {
-+  extends StreamTest with BeforeAndAfter with Matchers with IgnoreCometSuite {
- 
-   import testImplicits._
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
-index 746f289c393..0c99d028163 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
-@@ -25,10 +25,11 @@ import org.apache.spark.sql.catalyst.expressions
- import org.apache.spark.sql.catalyst.expressions._
- import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
- import org.apache.spark.sql.catalyst.types.DataTypeUtils
--import org.apache.spark.sql.execution.{FileSourceScanExec, SortExec, SparkPlan}
-+import org.apache.spark.sql.comet._
-+import org.apache.spark.sql.execution.{ColumnarToRowExec, FileSourceScanExec, SortExec, SparkPlan}
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper}
- import org.apache.spark.sql.execution.datasources.BucketingUtils
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
- import org.apache.spark.sql.execution.joins.SortMergeJoinExec
- import org.apache.spark.sql.functions._
- import org.apache.spark.sql.internal.SQLConf
-@@ -102,12 +103,20 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-     }
-   }
- 
--  private def getFileScan(plan: SparkPlan): FileSourceScanExec = {
--    val fileScan = collect(plan) { case f: FileSourceScanExec => f }
-+  private def getFileScan(plan: SparkPlan): SparkPlan = {
-+    val fileScan = collect(plan) {
-+      case f: FileSourceScanExec => f
-+      case f: CometScanExec => f
-+    }
-     assert(fileScan.nonEmpty, plan)
-     fileScan.head
-   }
- 
-+  private def getBucketScan(plan: SparkPlan): Boolean = getFileScan(plan) match {
-+    case fs: FileSourceScanExec => fs.bucketedScan
-+    case bs: CometScanExec => bs.bucketedScan
-+  }
-+
-   // To verify if the bucket pruning works, this function checks two conditions:
-   //   1) Check if the pruned buckets (before filtering) are empty.
-   //   2) Verify the final result is the same as the expected one
-@@ -156,7 +165,8 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-           val planWithoutBucketedScan = bucketedDataFrame.filter(filterCondition)
-             .queryExecution.executedPlan
-           val fileScan = getFileScan(planWithoutBucketedScan)
--          assert(!fileScan.bucketedScan, s"except no bucketed scan but found\n$fileScan")
-+          val bucketedScan = getBucketScan(planWithoutBucketedScan)
-+          assert(!bucketedScan, s"except no bucketed scan but found\n$fileScan")
- 
-           val bucketColumnType = bucketedDataFrame.schema.apply(bucketColumnIndex).dataType
-           val rowsWithInvalidBuckets = fileScan.execute().filter(row => {
-@@ -452,28 +462,49 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-         val joinOperator = if (joined.sqlContext.conf.adaptiveExecutionEnabled) {
-           val executedPlan =
-             joined.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
--          assert(executedPlan.isInstanceOf[SortMergeJoinExec])
--          executedPlan.asInstanceOf[SortMergeJoinExec]
-+          executedPlan match {
-+            case s: SortMergeJoinExec => s
-+            case b: CometSortMergeJoinExec =>
-+              b.originalPlan match {
-+                case s: SortMergeJoinExec => s
-+                case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+              }
-+            case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+          }
-         } else {
-           val executedPlan = joined.queryExecution.executedPlan
--          assert(executedPlan.isInstanceOf[SortMergeJoinExec])
--          executedPlan.asInstanceOf[SortMergeJoinExec]
-+          executedPlan match {
-+            case s: SortMergeJoinExec => s
-+            case ColumnarToRowExec(child) =>
-+              child.asInstanceOf[CometSortMergeJoinExec].originalPlan match {
-+                case s: SortMergeJoinExec => s
-+                case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+              }
-+            case CometColumnarToRowExec(child) =>
-+              child.asInstanceOf[CometSortMergeJoinExec].originalPlan match {
-+                case s: SortMergeJoinExec => s
-+                case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+              }
-+            case o => fail(s"expected SortMergeJoinExec, but found\n$o")
-+          }
-         }
- 
-         // check existence of shuffle
-         assert(
--          joinOperator.left.exists(_.isInstanceOf[ShuffleExchangeExec]) == shuffleLeft,
-+          joinOperator.left.exists(op => op.isInstanceOf[ShuffleExchangeLike]) == shuffleLeft,
-           s"expected shuffle in plan to be $shuffleLeft but found\n${joinOperator.left}")
-         assert(
--          joinOperator.right.exists(_.isInstanceOf[ShuffleExchangeExec]) == shuffleRight,
-+          joinOperator.right.exists(op => op.isInstanceOf[ShuffleExchangeLike]) == shuffleRight,
-           s"expected shuffle in plan to be $shuffleRight but found\n${joinOperator.right}")
- 
-         // check existence of sort
-         assert(
--          joinOperator.left.exists(_.isInstanceOf[SortExec]) == sortLeft,
-+          joinOperator.left.exists(op => op.isInstanceOf[SortExec] || op.isInstanceOf[CometExec] &&
-+            op.asInstanceOf[CometExec].originalPlan.isInstanceOf[SortExec]) == sortLeft,
-           s"expected sort in the left child to be $sortLeft but found\n${joinOperator.left}")
-         assert(
--          joinOperator.right.exists(_.isInstanceOf[SortExec]) == sortRight,
-+          joinOperator.right.exists(op => op.isInstanceOf[SortExec] || op.isInstanceOf[CometExec] &&
-+            op.asInstanceOf[CometExec].originalPlan.isInstanceOf[SortExec]) == sortRight,
-           s"expected sort in the right child to be $sortRight but found\n${joinOperator.right}")
- 
-         // check the output partitioning
-@@ -836,11 +867,11 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-       df1.write.format("parquet").bucketBy(8, "i").saveAsTable("bucketed_table")
- 
-       val scanDF = spark.table("bucketed_table").select("j")
--      assert(!getFileScan(scanDF.queryExecution.executedPlan).bucketedScan)
-+      assert(!getBucketScan(scanDF.queryExecution.executedPlan))
-       checkAnswer(scanDF, df1.select("j"))
- 
-       val aggDF = spark.table("bucketed_table").groupBy("j").agg(max("k"))
--      assert(!getFileScan(aggDF.queryExecution.executedPlan).bucketedScan)
-+      assert(!getBucketScan(aggDF.queryExecution.executedPlan))
-       checkAnswer(aggDF, df1.groupBy("j").agg(max("k")))
-     }
-   }
-@@ -1029,15 +1060,21 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
-           Seq(true, false).foreach { aqeEnabled =>
-             withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> aqeEnabled.toString) {
-               val plan = sql(query).queryExecution.executedPlan
--              val shuffles = collect(plan) { case s: ShuffleExchangeExec => s }
-+              val shuffles = collect(plan) { case s: ShuffleExchangeLike => s }
-               assert(shuffles.length == expectedNumShuffles)
- 
-               val scans = collect(plan) {
-                 case f: FileSourceScanExec if f.optionalNumCoalescedBuckets.isDefined => f
-+                case b: CometScanExec if b.optionalNumCoalescedBuckets.isDefined => b
-               }
-               if (expectedCoalescedNumBuckets.isDefined) {
-                 assert(scans.length == 1)
--                assert(scans.head.optionalNumCoalescedBuckets == expectedCoalescedNumBuckets)
-+                scans.head match {
-+                  case f: FileSourceScanExec =>
-+                    assert(f.optionalNumCoalescedBuckets == expectedCoalescedNumBuckets)
-+                  case b: CometScanExec =>
-+                    assert(b.optionalNumCoalescedBuckets == expectedCoalescedNumBuckets)
-+                }
-               } else {
-                 assert(scans.isEmpty)
-               }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
-index 6f897a9c0b7..b0723634f68 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
-@@ -20,6 +20,7 @@ package org.apache.spark.sql.sources
- import java.io.File
- 
- import org.apache.spark.SparkException
-+import org.apache.spark.sql.IgnoreCometSuite
- import org.apache.spark.sql.catalyst.TableIdentifier
- import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTableType}
- import org.apache.spark.sql.catalyst.parser.ParseException
-@@ -27,7 +28,10 @@ import org.apache.spark.sql.internal.SQLConf.BUCKETING_MAX_BUCKETS
- import org.apache.spark.sql.test.SharedSparkSession
- import org.apache.spark.util.Utils
- 
--class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession {
-+// For some reason this suite is flaky w/ or w/o Comet when running in Github workflow.
-+// Since it isn't related to Comet, we disable it for now.
-+class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession
-+    with IgnoreCometSuite {
-   import testImplicits._
- 
-   protected override lazy val sql = spark.sql _
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
-index d675503a8ba..659fa686fb7 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala
-@@ -18,6 +18,7 @@
- package org.apache.spark.sql.sources
- 
- import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.CometScanExec
- import org.apache.spark.sql.execution.FileSourceScanExec
- import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.internal.SQLConf
-@@ -68,7 +69,10 @@ abstract class DisableUnnecessaryBucketedScanSuite
- 
-     def checkNumBucketedScan(query: String, expectedNumBucketedScan: Int): Unit = {
-       val plan = sql(query).queryExecution.executedPlan
--      val bucketedScan = collect(plan) { case s: FileSourceScanExec if s.bucketedScan => s }
-+      val bucketedScan = collect(plan) {
-+        case s: FileSourceScanExec if s.bucketedScan => s
-+        case s: CometScanExec if s.bucketedScan => s
-+      }
-       assert(bucketedScan.length == expectedNumBucketedScan)
-     }
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
-index 1954cce7fdc..73d1464780e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
-@@ -34,6 +34,7 @@ import org.apache.spark.paths.SparkPath
- import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
- import org.apache.spark.sql.{AnalysisException, DataFrame}
- import org.apache.spark.sql.catalyst.util.stringToFile
-+import org.apache.spark.sql.comet.CometBatchScanExec
- import org.apache.spark.sql.execution.DataSourceScanExec
- import org.apache.spark.sql.execution.datasources._
- import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV2Relation, FileScan, FileTable}
-@@ -761,6 +762,8 @@ class FileStreamSinkV2Suite extends FileStreamSinkSuite {
-       val fileScan = df.queryExecution.executedPlan.collect {
-         case batch: BatchScanExec if batch.scan.isInstanceOf[FileScan] =>
-           batch.scan.asInstanceOf[FileScan]
-+        case batch: CometBatchScanExec if batch.scan.isInstanceOf[FileScan] =>
-+          batch.scan.asInstanceOf[FileScan]
-       }.headOption.getOrElse {
-         fail(s"No FileScan in query\n${df.queryExecution}")
-       }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-index b597a244710..b2e8be41065 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-@@ -21,6 +21,7 @@ import java.io.File
- 
- import org.apache.commons.io.FileUtils
- 
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.{FlatMapGroupsWithStateExec, MemoryStream}
- import org.apache.spark.sql.internal.SQLConf
-@@ -91,7 +92,7 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require StatefulOpClusteredDistribution " +
--    "from children - without initial state") {
-+    "from children - without initial state", IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -243,7 +244,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in 3.2.x - without initial state") {
-+    "from children if the query starts from checkpoint in 3.2.x - without initial state",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -335,7 +337,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in prior to 3.2") {
-+    "from children if the query starts from checkpoint in prior to 3.2",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-index a3774bf17e6..6879c71037d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-@@ -25,7 +25,7 @@ import org.scalatest.exceptions.TestFailedException
- 
- import org.apache.spark.SparkException
- import org.apache.spark.api.java.function.FlatMapGroupsWithStateFunction
--import org.apache.spark.sql.{DataFrame, Encoder}
-+import org.apache.spark.sql.{DataFrame, Encoder, IgnoreCometSuite}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow}
- import org.apache.spark.sql.catalyst.plans.logical.FlatMapGroupsWithState
-@@ -46,8 +46,9 @@ case class RunningCount(count: Long)
- 
- case class Result(key: Long, count: Int)
- 
-+// TODO: fix Comet to enable this suite
- @SlowSQLTest
--class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
-+class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest with IgnoreCometSuite {
- 
-   import testImplicits._
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-index 2a2a83d35e1..e3b7b290b3e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.streaming
- 
- import org.apache.spark.SparkException
--import org.apache.spark.sql.{AnalysisException, Dataset, KeyValueGroupedDataset}
-+import org.apache.spark.sql.{AnalysisException, Dataset, IgnoreComet, KeyValueGroupedDataset}
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.MemoryStream
- import org.apache.spark.sql.execution.streaming.state.FlatMapGroupsWithStateExecHelper
-@@ -253,7 +253,8 @@ class FlatMapGroupsWithStateWithInitialStateSuite extends StateStoreMetricsTest
-     assert(e.message.contains(expectedError))
-   }
- 
--  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState") {
-+  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     val initialStateDS = Seq(("keyInStateAndData", new RunningCount(1))).toDS()
-     val initialState: KeyValueGroupedDataset[String, RunningCount] =
-       initialStateDS.groupByKey(_._1).mapValues(_._2)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
-index c97979a57a5..45a998db0e0 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
-@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Range, RepartitionByExpressi
- import org.apache.spark.sql.catalyst.streaming.{InternalOutputModes, StreamingRelationV2}
- import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes
- import org.apache.spark.sql.catalyst.util.DateTimeUtils
-+import org.apache.spark.sql.comet.CometLocalLimitExec
- import org.apache.spark.sql.execution.{LocalLimitExec, SimpleMode, SparkPlan}
- import org.apache.spark.sql.execution.command.ExplainCommand
- import org.apache.spark.sql.execution.streaming._
-@@ -1114,11 +1115,12 @@ class StreamSuite extends StreamTest {
-       val localLimits = execPlan.collect {
-         case l: LocalLimitExec => l
-         case l: StreamingLocalLimitExec => l
-+        case l: CometLocalLimitExec => l
-       }
- 
-       require(
-         localLimits.size == 1,
--        s"Cant verify local limit optimization with this plan:\n$execPlan")
-+        s"Cant verify local limit optimization ${localLimits.size} with this plan:\n$execPlan")
- 
-       if (expectStreamingLimit) {
-         assert(
-@@ -1126,7 +1128,8 @@ class StreamSuite extends StreamTest {
-           s"Local limit was not StreamingLocalLimitExec:\n$execPlan")
-       } else {
-         assert(
--          localLimits.head.isInstanceOf[LocalLimitExec],
-+          localLimits.head.isInstanceOf[LocalLimitExec] ||
-+            localLimits.head.isInstanceOf[CometLocalLimitExec],
-           s"Local limit was not LocalLimitExec:\n$execPlan")
-       }
-     }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala
-index b4c4ec7acbf..20579284856 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala
-@@ -23,6 +23,7 @@ import org.apache.commons.io.FileUtils
- import org.scalatest.Assertions
- 
- import org.apache.spark.sql.catalyst.plans.physical.UnspecifiedDistribution
-+import org.apache.spark.sql.comet.CometHashAggregateExec
- import org.apache.spark.sql.execution.aggregate.BaseAggregateExec
- import org.apache.spark.sql.execution.streaming.{MemoryStream, StateStoreRestoreExec, StateStoreSaveExec}
- import org.apache.spark.sql.functions.count
-@@ -67,6 +68,7 @@ class StreamingAggregationDistributionSuite extends StreamTest
-         // verify aggregations in between, except partial aggregation
-         val allAggregateExecs = query.lastExecution.executedPlan.collect {
-           case a: BaseAggregateExec => a
-+          case c: CometHashAggregateExec => c.originalPlan
-         }
- 
-         val aggregateExecsWithoutPartialAgg = allAggregateExecs.filter {
-@@ -201,6 +203,7 @@ class StreamingAggregationDistributionSuite extends StreamTest
-         // verify aggregations in between, except partial aggregation
-         val allAggregateExecs = executedPlan.collect {
-           case a: BaseAggregateExec => a
-+          case c: CometHashAggregateExec => c.originalPlan
-         }
- 
-         val aggregateExecsWithoutPartialAgg = allAggregateExecs.filter {
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
-index aad91601758..201083bd621 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
-@@ -31,7 +31,7 @@ import org.apache.spark.scheduler.ExecutorCacheTaskLocation
- import org.apache.spark.sql.{DataFrame, Row, SparkSession}
- import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression}
- import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
--import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
-+import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
- import org.apache.spark.sql.execution.streaming.{MemoryStream, StatefulOperatorStateInfo, StreamingSymmetricHashJoinExec, StreamingSymmetricHashJoinHelper}
- import org.apache.spark.sql.execution.streaming.state.{RocksDBStateStoreProvider, StateStore, StateStoreProviderId}
- import org.apache.spark.sql.functions._
-@@ -619,14 +619,27 @@ class StreamingInnerJoinSuite extends StreamingJoinSuite {
- 
-         val numPartitions = spark.sqlContext.conf.getConf(SQLConf.SHUFFLE_PARTITIONS)
- 
--        assert(query.lastExecution.executedPlan.collect {
--          case j @ StreamingSymmetricHashJoinExec(_, _, _, _, _, _, _, _, _,
--            ShuffleExchangeExec(opA: HashPartitioning, _, _, _),
--            ShuffleExchangeExec(opB: HashPartitioning, _, _, _))
--              if partitionExpressionsColumns(opA.expressions) === Seq("a", "b")
--                && partitionExpressionsColumns(opB.expressions) === Seq("a", "b")
--                && opA.numPartitions == numPartitions && opB.numPartitions == numPartitions => j
--        }.size == 1)
-+        val join = query.lastExecution.executedPlan.collect {
-+          case j: StreamingSymmetricHashJoinExec => j
-+        }.head
-+        val opA = join.left.collect {
-+          case s: ShuffleExchangeLike
-+            if s.outputPartitioning.isInstanceOf[HashPartitioning] &&
-+              partitionExpressionsColumns(
-+                s.outputPartitioning
-+                  .asInstanceOf[HashPartitioning].expressions) === Seq("a", "b") =>
-+            s.outputPartitioning.asInstanceOf[HashPartitioning]
-+        }.head
-+        val opB = join.right.collect {
-+          case s: ShuffleExchangeLike
-+            if s.outputPartitioning.isInstanceOf[HashPartitioning] &&
-+              partitionExpressionsColumns(
-+                s.outputPartitioning
-+                  .asInstanceOf[HashPartitioning].expressions) === Seq("a", "b") =>
-+            s.outputPartitioning
-+              .asInstanceOf[HashPartitioning]
-+        }.head
-+        assert(opA.numPartitions == numPartitions && opB.numPartitions == numPartitions)
-       })
-   }
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
-index abe606ad9c1..2d930b64cca 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala
-@@ -22,7 +22,7 @@ import java.util
- 
- import org.scalatest.BeforeAndAfter
- 
--import org.apache.spark.sql.{AnalysisException, Row, SaveMode}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, Row, SaveMode}
- import org.apache.spark.sql.catalyst.TableIdentifier
- import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
- import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
-@@ -327,7 +327,8 @@ class DataStreamTableAPISuite extends StreamTest with BeforeAndAfter {
-     }
-   }
- 
--  test("explain with table on DSv1 data source") {
-+  test("explain with table on DSv1 data source",
-+      IgnoreComet("Comet explain output is different")) {
-     val tblSourceName = "tbl_src"
-     val tblTargetName = "tbl_target"
-     val tblSourceQualified = s"default.$tblSourceName"
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-index e937173a590..18c0232014b 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-@@ -27,6 +27,7 @@ import scala.concurrent.duration._
- import scala.language.implicitConversions
- import scala.util.control.NonFatal
- 
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.scalactic.source.Position
- import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
-@@ -41,6 +42,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
- import org.apache.spark.sql.catalyst.plans.PlanTestBase
- import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
- import org.apache.spark.sql.catalyst.util._
-+import org.apache.spark.sql.comet._
- import org.apache.spark.sql.execution.FilterExec
- import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
- import org.apache.spark.sql.execution.datasources.DataSourceUtils
-@@ -126,7 +128,26 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
-         }
-       }
-     } else {
--      super.test(testName, testTags: _*)(testFun)
-+      if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
-+        ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
-+      } else {
-+        val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
-+        val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT
-+        val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION
-+        if (isCometEnabled && isNativeIcebergCompat &&
-+          testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
-+          ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
-+        } else if (isCometEnabled && isNativeDataFusion &&
-+          testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
-+          ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
-+        } else if (isCometEnabled && (isNativeDataFusion || isNativeIcebergCompat) &&
-+          testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
-+          ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
-+            testTags: _*)(testFun)
-+        } else {
-+          super.test(testName, testTags: _*)(testFun)
-+        }
-+      }
-     }
-   }
- 
-@@ -242,6 +263,29 @@ private[sql] trait SQLTestUtilsBase
-     protected override def _sqlContext: SQLContext = self.spark.sqlContext
-   }
- 
-+  /**
-+   * Whether Comet extension is enabled
-+   */
-+  protected def isCometEnabled: Boolean = SparkSession.isCometEnabled
-+
-+  /**
-+   * Whether to enable ansi mode This is only effective when
-+   * [[isCometEnabled]] returns true.
-+   */
-+  protected def enableCometAnsiMode: Boolean = {
-+    val v = System.getenv("ENABLE_COMET_ANSI_MODE")
-+    v != null && v.toBoolean
-+  }
-+
-+  /**
-+   * Whether Spark should only apply Comet scan optimization. This is only effective when
-+   * [[isCometEnabled]] returns true.
-+   */
-+  protected def isCometScanOnly: Boolean = {
-+    val v = System.getenv("ENABLE_COMET_SCAN_ONLY")
-+    v != null && v.toBoolean
-+  }
-+
-   protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
-     SparkSession.setActiveSession(spark)
-     super.withSQLConf(pairs: _*)(f)
-@@ -435,6 +479,8 @@ private[sql] trait SQLTestUtilsBase
-     val schema = df.schema
-     val withoutFilters = df.queryExecution.executedPlan.transform {
-       case FilterExec(_, child) => child
-+      case CometFilterExec(_, _, _, _, child, _) => child
-+      case CometProjectExec(_, _, _, _, CometFilterExec(_, _, _, _, child, _), _) => child
-     }
- 
-     spark.internalCreateDataFrame(withoutFilters.execute(), schema)
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-index ed2e309fa07..71ba6533c9d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-@@ -74,6 +74,31 @@ trait SharedSparkSessionBase
-       // this rule may potentially block testing of other optimization rules such as
-       // ConstantPropagation etc.
-       .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)
-+    // Enable Comet if `ENABLE_COMET` environment variable is set
-+    if (isCometEnabled) {
-+      conf
-+        .set("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions")
-+        .set("spark.comet.enabled", "true")
-+
-+      if (!isCometScanOnly) {
-+        conf
-+          .set("spark.comet.exec.enabled", "true")
-+          .set("spark.shuffle.manager",
-+            "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
-+          .set("spark.comet.exec.shuffle.enabled", "true")
-+          .set("spark.comet.memoryOverhead", "10g")
-+      } else {
-+        conf
-+          .set("spark.comet.exec.enabled", "false")
-+          .set("spark.comet.exec.shuffle.enabled", "false")
-+      }
-+
-+      if (enableCometAnsiMode) {
-+        conf
-+          .set("spark.sql.ansi.enabled", "true")
-+          .set("spark.comet.ansi.enabled", "true")
-+      }
-+    }
-     conf.set(
-       StaticSQLConf.WAREHOUSE_PATH,
-       conf.get(StaticSQLConf.WAREHOUSE_PATH) + "/" + getClass.getCanonicalName)
-diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
-index c63c748953f..7edca9c93a6 100644
---- a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
-@@ -45,7 +45,7 @@ class SqlResourceWithActualMetricsSuite
-   import testImplicits._
- 
-   // Exclude nodes which may not have the metrics
--  val excludedNodes = List("WholeStageCodegen", "Project", "SerializeFromObject")
-+  val excludedNodes = List("WholeStageCodegen", "Project", "SerializeFromObject", "RowToColumnar")
- 
-   implicit val formats = new DefaultFormats {
-     override def dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
-diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala
-index 52abd248f3a..7a199931a08 100644
---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala
-+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/DynamicPartitionPruningHiveScanSuite.scala
-@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive
- 
- import org.apache.spark.sql._
- import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
-+import org.apache.spark.sql.comet._
- import org.apache.spark.sql.execution._
- import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
- import org.apache.spark.sql.hive.execution.HiveTableScanExec
-@@ -35,6 +36,9 @@ abstract class DynamicPartitionPruningHiveScanSuiteBase
-       case s: FileSourceScanExec => s.partitionFilters.collect {
-         case d: DynamicPruningExpression => d.child
-       }
-+      case s: CometScanExec => s.partitionFilters.collect {
-+        case d: DynamicPruningExpression => d.child
-+      }
-       case h: HiveTableScanExec => h.partitionPruningPred.collect {
-         case d: DynamicPruningExpression => d.child
-       }
-diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
-index 549431ef4f4..e48f1730da6 100644
---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
-+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
-@@ -22,7 +22,7 @@ import java.io.RandomAccessFile
- import java.nio.charset.StandardCharsets
- import java.util.Base64
- 
--import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.{IgnoreCometNativeScan, QueryTest}
- import org.apache.spark.sql.hive.test.TestHiveSingleton
- 
- /**
-@@ -37,7 +37,8 @@ class ParquetEncryptionSuite extends QueryTest with TestHiveSingleton {
-   private val key1 = encoder.encodeToString("1234567890123450".getBytes(StandardCharsets.UTF_8))
-   private val key2 = encoder.encodeToString("1234567890123451".getBytes(StandardCharsets.UTF_8))
- 
--  test("SPARK-34990: Write and read an encrypted parquet") {
-+  test("SPARK-34990: Write and read an encrypted parquet",
-+    IgnoreCometNativeScan("no encryption support yet")) {
-     withTempDir { dir =>
-       withSQLConf(
-         "parquet.crypto.factory.class" ->
-@@ -64,7 +65,8 @@ class ParquetEncryptionSuite extends QueryTest with TestHiveSingleton {
-     }
-   }
- 
--  test("SPARK-37117: Can't read files in Parquet encryption external key material mode") {
-+  test("SPARK-37117: Can't read files in Parquet encryption external key material mode",
-+    IgnoreCometNativeScan("no encryption support yet")) {
-     withTempDir { dir =>
-       withSQLConf(
-         "parquet.crypto.factory.class" ->
-@@ -91,7 +93,8 @@ class ParquetEncryptionSuite extends QueryTest with TestHiveSingleton {
-     }
-   }
- 
--  test("SPARK-42114: Test of uniform parquet encryption") {
-+  test("SPARK-42114: Test of uniform parquet encryption",
-+    IgnoreCometNativeScan("no encryption support yet")) {
-     withTempDir { dir =>
-       withSQLConf(
-         "parquet.crypto.factory.class" ->
-diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-index 1d646f40b3e..7f2cdb8f061 100644
---- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-@@ -53,25 +53,55 @@ object TestHive
-     new SparkContext(
-       System.getProperty("spark.sql.test.master", "local[1]"),
-       "TestSQLContext",
--      new SparkConf()
--        .set("spark.sql.test", "")
--        .set(SQLConf.CODEGEN_FALLBACK.key, "false")
--        .set(SQLConf.CODEGEN_FACTORY_MODE.key, CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
--        .set(HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key,
--          "org.apache.spark.sql.hive.execution.PairSerDe")
--        .set(WAREHOUSE_PATH.key, TestHiveContext.makeWarehouseDir().toURI.getPath)
--        // SPARK-8910
--        .set(UI_ENABLED, false)
--        .set(config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
--        // Hive changed the default of hive.metastore.disallow.incompatible.col.type.changes
--        // from false to true. For details, see the JIRA HIVE-12320 and HIVE-17764.
--        .set("spark.hadoop.hive.metastore.disallow.incompatible.col.type.changes", "false")
--        // Disable ConvertToLocalRelation for better test coverage. Test cases built on
--        // LocalRelation will exercise the optimization rules better by disabling it as
--        // this rule may potentially block testing of other optimization rules such as
--        // ConstantPropagation etc.
--        .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)))
-+      {
-+        val conf = new SparkConf()
-+          .set("spark.sql.test", "")
-+          .set(SQLConf.CODEGEN_FALLBACK.key, "false")
-+          .set(SQLConf.CODEGEN_FACTORY_MODE.key, CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
-+          .set(HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key,
-+            "org.apache.spark.sql.hive.execution.PairSerDe")
-+          .set(WAREHOUSE_PATH.key, TestHiveContext.makeWarehouseDir().toURI.getPath)
-+          // SPARK-8910
-+          .set(UI_ENABLED, false)
-+          .set(config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
-+          // Hive changed the default of hive.metastore.disallow.incompatible.col.type.changes
-+          // from false to true. For details, see the JIRA HIVE-12320 and HIVE-17764.
-+          .set("spark.hadoop.hive.metastore.disallow.incompatible.col.type.changes", "false")
-+          // Disable ConvertToLocalRelation for better test coverage. Test cases built on
-+          // LocalRelation will exercise the optimization rules better by disabling it as
-+          // this rule may potentially block testing of other optimization rules such as
-+          // ConstantPropagation etc.
-+          .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)
-+
-+        if (SparkSession.isCometEnabled) {
-+          conf
-+            .set("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions")
-+            .set("spark.comet.enabled", "true")
-+
-+          val v = System.getenv("ENABLE_COMET_SCAN_ONLY")
-+          if (v == null || !v.toBoolean) {
-+            conf
-+              .set("spark.comet.exec.enabled", "true")
-+              .set("spark.shuffle.manager",
-+                "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
-+              .set("spark.comet.exec.shuffle.enabled", "true")
-+          } else {
-+            conf
-+              .set("spark.comet.exec.enabled", "false")
-+              .set("spark.comet.exec.shuffle.enabled", "false")
-+          }
-+
-+          val a = System.getenv("ENABLE_COMET_ANSI_MODE")
-+          if (a != null && a.toBoolean) {
-+            conf
-+              .set("spark.sql.ansi.enabled", "true")
-+              .set("spark.comet.ansi.enabled", "true")
-+          }
-+        }
- 
-+        conf
-+      }
-+    ))
- 
- case class TestHiveVersion(hiveClient: HiveClient)
-   extends TestHiveContext(TestHive.sparkContext, hiveClient)
diff --git a/dev/diffs/3.5.6.diff b/dev/diffs/3.5.6.diff
index e3ba0a35c3..4897de62a7 100644
--- a/dev/diffs/3.5.6.diff
+++ b/dev/diffs/3.5.6.diff
@@ -263,7 +263,7 @@ index 56e9520fdab..917932336df 100644
            spark.range(100).write.saveAsTable(s"$dbName.$table2Name")
  
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-index 7ee18df3756..64f01a68048 100644
+index 7ee18df3756..d09f70e5d99 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
 @@ -40,11 +40,12 @@ import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
@@ -322,16 +322,6 @@ index 7ee18df3756..64f01a68048 100644
      }
      assert(exchanges.size == 2)
    }
-@@ -3350,7 +3353,8 @@ class DataFrameSuite extends QueryTest
-     assert(df2.isLocal)
-   }
- 
--  test("SPARK-35886: PromotePrecision should be subexpr replaced") {
-+  test("SPARK-35886: PromotePrecision should be subexpr replaced",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     withTable("tbl") {
-       sql(
-         """
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
 index 47a311c71d5..342e71cfdd4 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
@@ -889,37 +879,6 @@ index 793a0da6a86..6ccb9d62582 100644
      AccumulatorSuite.verifyPeakExecutionMemorySet(sparkContext, "external sort") {
        sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC").collect()
      }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-index 8b4ac474f87..3f79f20822f 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
-@@ -223,6 +223,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     withSession(extensions) { session =>
-       session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true)
-       session.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1")
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      session.conf.set("spark.comet.enabled", false)
-       assert(session.sessionState.columnarRules.contains(
-         MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
-       import session.sqlContext.implicits._
-@@ -281,6 +283,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     }
-     withSession(extensions) { session =>
-       session.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE)
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      session.conf.set("spark.comet.enabled", false)
-       assert(session.sessionState.columnarRules.contains(
-         MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
-       import session.sqlContext.implicits._
-@@ -319,6 +323,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite with SQLHelper with Adapt
-     val session = SparkSession.builder()
-       .master("local[1]")
-       .config(COLUMN_BATCH_SIZE.key, 2)
-+      // https://github.com/apache/datafusion-comet/issues/1197
-+      .config("spark.comet.enabled", false)
-       .withExtensions { extensions =>
-         extensions.injectColumnar(session =>
-           MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) }
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
 index fa1a64460fc..1d2e215d6a3 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -1821,39 +1780,6 @@ index 2f8e401e743..a4f94417dcc 100644
        plan.inputPlan.output.zip(plan.finalPhysicalPlan.output).foreach { case (o1, o2) =>
          assert(o1.semanticEquals(o2), "Different output column order after AQE optimization")
        }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-index 05872d41131..a2c328b9742 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCustomMetadataStructSuite.scala
-@@ -21,7 +21,7 @@ import java.io.File
- 
- import org.apache.hadoop.fs.{FileStatus, Path}
- 
--import org.apache.spark.sql.{DataFrame, Dataset, QueryTest, Row}
-+import org.apache.spark.sql.{DataFrame, Dataset, IgnoreComet, QueryTest, Row}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{Expression, FileSourceConstantMetadataStructField, FileSourceGeneratedMetadataStructField, Literal}
- import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
-@@ -134,7 +134,8 @@ class FileSourceCustomMetadataStructSuite extends QueryTest with SharedSparkSess
-     }
-   }
- 
--  test("[SPARK-43226] extra constant metadata fields with extractors") {
-+  test("[SPARK-43226] extra constant metadata fields with extractors",
-+      IgnoreComet("TODO: fix Comet for this test")) {
-     withTempData("parquet", FILE_SCHEMA) { (_, f0, f1) =>
-       val format = new TestFileFormat(extraConstantMetadataFields) {
-         val extractPartitionNumber = { pf: PartitionedFile =>
-@@ -335,7 +336,8 @@ class FileSourceCustomMetadataStructSuite extends QueryTest with SharedSparkSess
-     }
-   }
- 
--  test("generated columns and extractors take precedence over metadata map values") {
-+  test("generated columns and extractors take precedence over metadata map values",
-+      IgnoreComet("TODO: fix Comet for this test")) {
-     withTempData("parquet", FILE_SCHEMA) { (_, f0, f1) =>
-       import FileFormat.{FILE_NAME, FILE_SIZE}
-       import ParquetFileFormat.ROW_INDEX
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
 index bf496d6db21..9bb57a9b4c6 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -1951,7 +1877,7 @@ index 07e2849ce6f..3e73645b638 100644
        ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
      )
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 8e88049f51e..5a963dab6f2 100644
+index 8e88049f51e..d3c0737d52e 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
 @@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -2012,17 +1938,7 @@ index 8e88049f51e..5a963dab6f2 100644
      import testImplicits._
      // keep() should take effect on StartsWith/EndsWith/Contains
      Seq(
-@@ -1743,7 +1756,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
-+  test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+      IgnoreComet("IN predicate is not yet supported in Comet, see issue #36")) {
-     val schema = StructType(Seq(
-       StructField("a", IntegerType, nullable = false)
-     ))
-@@ -1984,7 +1998,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
+@@ -1984,7 +1997,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
      }
    }
  
@@ -2032,7 +1948,7 @@ index 8e88049f51e..5a963dab6f2 100644
      // block 1:
      //                      null count  min                                       max
      // page-0                         0  0                                         99
-@@ -2044,7 +2059,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
+@@ -2044,7 +2058,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
      }
    }
  
@@ -2042,7 +1958,7 @@ index 8e88049f51e..5a963dab6f2 100644
      withTempPath { dir =>
        val path = dir.getCanonicalPath
        spark.range(100).selectExpr("id * 2 AS id")
-@@ -2276,7 +2292,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
+@@ -2276,7 +2291,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
            assert(pushedParquetFilters.exists(_.getClass === filterClass),
              s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
  
@@ -2055,7 +1971,7 @@ index 8e88049f51e..5a963dab6f2 100644
          } else {
            assert(selectedFilters.isEmpty, "There is filter pushed down")
          }
-@@ -2336,7 +2356,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
+@@ -2336,7 +2355,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
            assert(pushedParquetFilters.exists(_.getClass === filterClass),
              s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
  
@@ -2577,94 +2493,6 @@ index 1954cce7fdc..73d1464780e 100644
        }.headOption.getOrElse {
          fail(s"No FileScan in query\n${df.queryExecution}")
        }
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-index b597a244710..b2e8be41065 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala
-@@ -21,6 +21,7 @@ import java.io.File
- 
- import org.apache.commons.io.FileUtils
- 
-+import org.apache.spark.sql.IgnoreComet
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.{FlatMapGroupsWithStateExec, MemoryStream}
- import org.apache.spark.sql.internal.SQLConf
-@@ -91,7 +92,7 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require StatefulOpClusteredDistribution " +
--    "from children - without initial state") {
-+    "from children - without initial state", IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -243,7 +244,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in 3.2.x - without initial state") {
-+    "from children if the query starts from checkpoint in 3.2.x - without initial state",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-@@ -335,7 +337,8 @@ class FlatMapGroupsWithStateDistributionSuite extends StreamTest
-   }
- 
-   test("SPARK-38204: flatMapGroupsWithState should require ClusteredDistribution " +
--    "from children if the query starts from checkpoint in prior to 3.2") {
-+    "from children if the query starts from checkpoint in prior to 3.2",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     // function will return -1 on timeout and returns count of the state otherwise
-     val stateFunc =
-       (key: (String, String), values: Iterator[(String, String, Long)],
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-index a3774bf17e6..6879c71037d 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
-@@ -25,7 +25,7 @@ import org.scalatest.exceptions.TestFailedException
- 
- import org.apache.spark.SparkException
- import org.apache.spark.api.java.function.FlatMapGroupsWithStateFunction
--import org.apache.spark.sql.{DataFrame, Encoder}
-+import org.apache.spark.sql.{DataFrame, Encoder, IgnoreCometSuite}
- import org.apache.spark.sql.catalyst.InternalRow
- import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeProjection, UnsafeRow}
- import org.apache.spark.sql.catalyst.plans.logical.FlatMapGroupsWithState
-@@ -46,8 +46,9 @@ case class RunningCount(count: Long)
- 
- case class Result(key: Long, count: Int)
- 
-+// TODO: fix Comet to enable this suite
- @SlowSQLTest
--class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
-+class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest with IgnoreCometSuite {
- 
-   import testImplicits._
- 
-diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-index 2a2a83d35e1..e3b7b290b3e 100644
---- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateWithInitialStateSuite.scala
-@@ -18,7 +18,7 @@
- package org.apache.spark.sql.streaming
- 
- import org.apache.spark.SparkException
--import org.apache.spark.sql.{AnalysisException, Dataset, KeyValueGroupedDataset}
-+import org.apache.spark.sql.{AnalysisException, Dataset, IgnoreComet, KeyValueGroupedDataset}
- import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Update
- import org.apache.spark.sql.execution.streaming.MemoryStream
- import org.apache.spark.sql.execution.streaming.state.FlatMapGroupsWithStateExecHelper
-@@ -253,7 +253,8 @@ class FlatMapGroupsWithStateWithInitialStateSuite extends StateStoreMetricsTest
-     assert(e.message.contains(expectedError))
-   }
- 
--  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState") {
-+  test("flatMapGroupsWithState - initial state - initial state has flatMapGroupsWithState",
-+    IgnoreComet("TODO: fix Comet for this test")) {
-     val initialStateDS = Seq(("keyInStateAndData", new RunningCount(1))).toDS()
-     val initialState: KeyValueGroupedDataset[String, RunningCount] =
-       initialStateDS.groupByKey(_._1).mapValues(_._2)
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
 index c97979a57a5..45a998db0e0 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff
index a59c85c212..f4cb601e41 100644
--- a/dev/diffs/4.0.0-preview1.diff
+++ b/dev/diffs/4.0.0-preview1.diff
@@ -2098,7 +2098,7 @@ index cd6f41b4ef4..4b6a17344bc 100644
        ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
      )
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 795e9f46a8d..f82c60c5433 100644
+index 795e9f46a8d..1797084a941 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
 @@ -1100,7 +1100,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -2150,17 +2150,7 @@ index 795e9f46a8d..f82c60c5433 100644
        }
      }
    }
-@@ -1748,7 +1761,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
-     }
-   }
- 
--  test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
-+  test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+      IgnoreComet("IN predicate is not yet supported in Comet, see issue #36")) {
-     val schema = StructType(Seq(
-       StructField("a", IntegerType, nullable = false)
-     ))
-@@ -1991,7 +2005,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
+@@ -1991,7 +2004,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
      }
    }
  
@@ -2170,7 +2160,7 @@ index 795e9f46a8d..f82c60c5433 100644
      // block 1:
      //                      null count  min                                       max
      // page-0                         0  0                                         99
-@@ -2051,7 +2066,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
+@@ -2051,7 +2065,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
      }
    }
  
@@ -2180,7 +2170,7 @@ index 795e9f46a8d..f82c60c5433 100644
      withTempPath { dir =>
        val path = dir.getCanonicalPath
        spark.range(100).selectExpr("id * 2 AS id")
-@@ -2301,7 +2317,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
+@@ -2301,7 +2316,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
            assert(pushedParquetFilters.exists(_.getClass === filterClass),
              s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
  
@@ -2193,7 +2183,7 @@ index 795e9f46a8d..f82c60c5433 100644
          } else {
            assert(selectedFilters.isEmpty, "There is filter pushed down")
          }
-@@ -2362,7 +2382,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
+@@ -2362,7 +2381,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
            assert(pushedParquetFilters.exists(_.getClass === filterClass),
              s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
  
diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md
index 256cd5416c..3e9847618a 100644
--- a/docs/source/user-guide/installation.md
+++ b/docs/source/user-guide/installation.md
@@ -37,23 +37,21 @@ Comet versions.
 We recommend only using Comet with Spark versions where we currently have both Comet and Spark tests enabled in CI.
 Other versions may work well enough for development and evaluation purposes.
 
-| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI |
-|---------------| ------------ | ------------- | ----------------- | ----------------- |
-| 3.4.3         | 11/17        | 2.12/2.13     | Yes               | Yes               |
-| 3.5.2         | 11/17        | 2.12/2.13     | Partial\*         | No                |
-| 3.5.3         | 11/17        | 2.12/2.13     | Partial\*         | No                |
-| 3.5.4         | 11/17        | 2.12/2.13     | Partial\*         | Yes               |
-| 3.5.5         | 11/17        | 2.12/2.13     | Partial\*         | Yes               |
-| 3.5.6         | 11/17        | 2.12/2.13     | Yes               | Yes               |
+| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
+|---------------| ------------ | ------------- |-------------------|-----------------------|
+| 3.4.3         | 11/17        | 2.12/2.13     | Yes               | Yes                   |
+| 3.5.4         | 11/17        | 2.12/2.13     | Yes               | No                    |
+| 3.5.5         | 11/17        | 2.12/2.13     | Yes               | No                    |
+| 3.5.6         | 11/17        | 2.12/2.13     | Yes               | Yes                   |
 
-\* For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI.
+Note that we do not test the full matrix of supported Java and Scala versions in CI for every Spark version.
 
 Experimental support is provided for the following versions of Apache Spark and is intended for development/testing
 use only and should not be used in production yet.
 
-| Spark Version  | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI |
-| -------------- | ------------ | ------------- | ----------------- | ----------------- |
-| 4.0.0-preview1 | 17           | 2.13          | Yes               | Yes               |
+| Spark Version  | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
+| -------------- | ------------ | ------------- | ----------------- |-----------------------|
+| 4.0.0-preview1 | 17           | 2.13          | Yes               | Yes                   |
 
 Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by
 Cloud Service Providers.
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 6fea0b84ee..71662f97fa 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -1265,7 +1265,7 @@ dependencies = [
 [[package]]
 name = "datafusion"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -1273,29 +1273,29 @@ dependencies = [
  "async-trait",
  "bytes",
  "chrono",
- "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-catalog",
  "datafusion-catalog-listing",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
  "datafusion-datasource-csv",
  "datafusion-datasource-json",
  "datafusion-datasource-parquet",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions",
  "datafusion-functions-aggregate",
  "datafusion-functions-nested",
  "datafusion-functions-table",
  "datafusion-functions-window",
  "datafusion-optimizer",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
  "datafusion-physical-optimizer",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "datafusion-sql",
  "futures",
  "itertools 0.14.0",
  "log",
@@ -1311,31 +1311,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "datafusion-catalog"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "arrow",
- "async-trait",
- "dashmap",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
- "parking_lot",
- "tokio",
-]
-
 [[package]]
 name = "datafusion-catalog"
 version = "48.0.0"
@@ -1344,15 +1319,15 @@ dependencies = [
  "arrow",
  "async-trait",
  "dashmap",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "datafusion-sql",
  "futures",
  "itertools 0.14.0",
  "log",
@@ -1364,19 +1339,19 @@ dependencies = [
 [[package]]
 name = "datafusion-catalog-listing"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "async-trait",
- "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
  "futures",
  "log",
  "object_store",
@@ -1472,28 +1447,6 @@ dependencies = [
  "twox-hash 2.1.0",
 ]
 
-[[package]]
-name = "datafusion-common"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "ahash",
- "arrow",
- "arrow-ipc",
- "base64",
- "half",
- "hashbrown 0.14.5",
- "indexmap",
- "libc",
- "log",
- "object_store",
- "parquet",
- "paste",
- "sqlparser",
- "tokio",
- "web-time",
-]
-
 [[package]]
 name = "datafusion-common"
 version = "48.0.0"
@@ -1509,22 +1462,13 @@ dependencies = [
  "libc",
  "log",
  "object_store",
+ "parquet",
  "paste",
  "sqlparser",
  "tokio",
  "web-time",
 ]
 
-[[package]]
-name = "datafusion-common-runtime"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "futures",
- "log",
- "tokio",
-]
-
 [[package]]
 name = "datafusion-common-runtime"
 version = "48.0.0"
@@ -1538,20 +1482,20 @@ dependencies = [
 [[package]]
 name = "datafusion-datasource"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
  "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
  "futures",
  "glob",
  "itertools 0.14.0",
@@ -1564,51 +1508,24 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "datafusion-datasource"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
-dependencies = [
- "arrow",
- "async-trait",
- "bytes",
- "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "futures",
- "glob",
- "itertools 0.14.0",
- "log",
- "object_store",
- "rand 0.9.1",
- "tokio",
- "url",
-]
-
 [[package]]
 name = "datafusion-datasource-csv"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
- "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
  "futures",
  "object_store",
  "regex",
@@ -1618,21 +1535,21 @@ dependencies = [
 [[package]]
 name = "datafusion-datasource-json"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
- "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
  "futures",
  "object_store",
  "serde_json",
@@ -1642,23 +1559,23 @@ dependencies = [
 [[package]]
 name = "datafusion-datasource-parquet"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
- "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-datasource 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
  "datafusion-functions-aggregate",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
  "datafusion-physical-optimizer",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-session 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-physical-plan",
+ "datafusion-session",
  "futures",
  "itertools 0.14.0",
  "log",
@@ -1669,34 +1586,11 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "datafusion-doc"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-
 [[package]]
 name = "datafusion-doc"
 version = "48.0.0"
 source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 
-[[package]]
-name = "datafusion-execution"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "arrow",
- "dashmap",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "futures",
- "log",
- "object_store",
- "parking_lot",
- "rand 0.9.1",
- "tempfile",
- "url",
-]
-
 [[package]]
 name = "datafusion-execution"
 version = "48.0.0"
@@ -1704,8 +1598,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7
 dependencies = [
  "arrow",
  "dashmap",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-expr",
  "futures",
  "log",
  "object_store",
@@ -1715,25 +1609,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "datafusion-expr"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "arrow",
- "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "indexmap",
- "paste",
- "serde_json",
- "sqlparser",
-]
-
 [[package]]
 name = "datafusion-expr"
 version = "48.0.0"
@@ -1741,37 +1616,25 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7
 dependencies = [
  "arrow",
  "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-window-common",
+ "datafusion-physical-expr-common",
  "indexmap",
  "paste",
  "serde_json",
  "sqlparser",
 ]
 
-[[package]]
-name = "datafusion-expr-common"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "indexmap",
- "itertools 0.14.0",
- "paste",
-]
-
 [[package]]
 name = "datafusion-expr-common"
 version = "48.0.0"
 source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
  "indexmap",
  "itertools 0.14.0",
  "paste",
@@ -1780,7 +1643,7 @@ dependencies = [
 [[package]]
 name = "datafusion-functions"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -1788,12 +1651,12 @@ dependencies = [
  "blake2",
  "blake3",
  "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-macros",
  "hex",
  "itertools 0.14.0",
  "log",
@@ -1805,62 +1668,26 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "datafusion-functions"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
-dependencies = [
- "arrow",
- "arrow-buffer",
- "base64",
- "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "hex",
- "itertools 0.14.0",
- "log",
- "rand 0.9.1",
- "regex",
- "unicode-segmentation",
- "uuid",
-]
-
 [[package]]
 name = "datafusion-functions-aggregate"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "ahash",
  "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions-aggregate-common",
+ "datafusion-macros",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
  "half",
  "log",
  "paste",
 ]
 
-[[package]]
-name = "datafusion-functions-aggregate-common"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "ahash",
- "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
-]
-
 [[package]]
 name = "datafusion-functions-aggregate-common"
 version = "48.0.0"
@@ -1868,26 +1695,26 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7
 dependencies = [
  "ahash",
  "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-expr-common",
+ "datafusion-physical-expr-common",
 ]
 
 [[package]]
 name = "datafusion-functions-nested"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "arrow-ord",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
  "datafusion-functions-aggregate",
- "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-macros",
+ "datafusion-physical-expr-common",
  "itertools 0.14.0",
  "log",
  "paste",
@@ -1896,14 +1723,14 @@ dependencies = [
 [[package]]
 name = "datafusion-functions-table"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "async-trait",
- "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-physical-plan",
  "parking_lot",
  "paste",
 ]
@@ -1911,46 +1738,27 @@ dependencies = [
 [[package]]
 name = "datafusion-functions-window"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-expr",
+ "datafusion-functions-window-common",
+ "datafusion-macros",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
  "log",
  "paste",
 ]
 
-[[package]]
-name = "datafusion-functions-window-common"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
-]
-
 [[package]]
 name = "datafusion-functions-window-common"
 version = "48.0.0"
 source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
-]
-
-[[package]]
-name = "datafusion-macros"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "quote",
- "syn 2.0.101",
+ "datafusion-common",
+ "datafusion-physical-expr-common",
 ]
 
 [[package]]
@@ -1958,7 +1766,7 @@ name = "datafusion-macros"
 version = "48.0.0"
 source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-expr",
  "quote",
  "syn 2.0.101",
 ]
@@ -1966,13 +1774,13 @@ dependencies = [
 [[package]]
 name = "datafusion-optimizer"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
  "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-physical-expr",
  "indexmap",
  "itertools 0.14.0",
  "log",
@@ -1980,27 +1788,6 @@ dependencies = [
  "regex-syntax",
 ]
 
-[[package]]
-name = "datafusion-physical-expr"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "ahash",
- "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "half",
- "hashbrown 0.14.5",
- "indexmap",
- "itertools 0.14.0",
- "log",
- "paste",
- "petgraph 0.8.1",
-]
-
 [[package]]
 name = "datafusion-physical-expr"
 version = "48.0.0"
@@ -2008,11 +1795,11 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7
 dependencies = [
  "ahash",
  "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr-common",
  "half",
  "hashbrown 0.14.5",
  "indexmap",
@@ -2022,19 +1809,6 @@ dependencies = [
  "petgraph 0.8.1",
 ]
 
-[[package]]
-name = "datafusion-physical-expr-common"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "ahash",
- "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "hashbrown 0.14.5",
- "itertools 0.14.0",
-]
-
 [[package]]
 name = "datafusion-physical-expr-common"
 version = "48.0.0"
@@ -2042,8 +1816,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7
 dependencies = [
  "ahash",
  "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-expr-common",
  "hashbrown 0.14.5",
  "itertools 0.14.0",
 ]
@@ -2051,47 +1825,18 @@ dependencies = [
 [[package]]
 name = "datafusion-physical-optimizer"
 version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "arrow",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "itertools 0.14.0",
- "log",
-]
-
-[[package]]
-name = "datafusion-physical-plan"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
+source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
- "ahash",
  "arrow",
- "arrow-ord",
- "arrow-schema",
- "async-trait",
- "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "futures",
- "half",
- "hashbrown 0.14.5",
- "indexmap",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
  "itertools 0.14.0",
  "log",
- "parking_lot",
- "pin-project-lite",
- "tokio",
 ]
 
 [[package]]
@@ -2105,13 +1850,13 @@ dependencies = [
  "arrow-schema",
  "async-trait",
  "chrono",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions-window-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
  "futures",
  "half",
  "hashbrown 0.14.5",
@@ -2123,29 +1868,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "datafusion-session"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "arrow",
- "async-trait",
- "dashmap",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
- "parking_lot",
- "tokio",
-]
-
 [[package]]
 name = "datafusion-session"
 version = "48.0.0"
@@ -2154,13 +1876,13 @@ dependencies = [
  "arrow",
  "async-trait",
  "dashmap",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-common-runtime 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-physical-plan 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-sql 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-plan",
+ "datafusion-sql",
  "futures",
  "itertools 0.14.0",
  "log",
@@ -2175,28 +1897,13 @@ version = "48.0.0"
 source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7e3c705d0f55d05c24a115a2f98"
 dependencies = [
  "arrow",
- "datafusion-catalog 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "log",
-]
-
-[[package]]
-name = "datafusion-sql"
-version = "48.0.0"
-source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3"
-dependencies = [
- "arrow",
- "bigdecimal",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc2)",
- "indexmap",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-macros",
  "log",
- "regex",
- "sqlparser",
 ]
 
 [[package]]
@@ -2206,8 +1913,8 @@ source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc3#33a32d4382bee7
 dependencies = [
  "arrow",
  "bigdecimal",
- "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
- "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=48.0.0-rc3)",
+ "datafusion-common",
+ "datafusion-expr",
  "indexmap",
  "log",
  "regex",