diff --git a/docs/source/user-guide/compatibility.md b/docs/source/user-guide/compatibility.md index 6e9f0f8253..4316c1c8cd 100644 --- a/docs/source/user-guide/compatibility.md +++ b/docs/source/user-guide/compatibility.md @@ -62,7 +62,6 @@ logical types. Arrow-based readers, such as DataFusion and Comet do respect thes rather than signed. By default, Comet will fall back to Spark when scanning Parquet files containing `byte` or `short` types (regardless of the logical type). This behavior can be disabled by setting `spark.comet.scan.allowIncompatible=true`. -- Reading legacy INT96 timestamps contained within complex types can produce different results to Spark - There is a known performance issue when pushing filters down to Parquet. See the [Comet Tuning Guide] for more information. - There are failures in the Spark SQL test suite when enabling these new scans (tracking issues: [#1542] and [#1545]). diff --git a/docs/templates/compatibility-template.md b/docs/templates/compatibility-template.md index a963824540..1915073858 100644 --- a/docs/templates/compatibility-template.md +++ b/docs/templates/compatibility-template.md @@ -62,7 +62,6 @@ The new scans currently have the following limitations: rather than signed. By default, Comet will fall back to Spark when scanning Parquet files containing `byte` or `short` types (regardless of the logical type). This behavior can be disabled by setting `spark.comet.scan.allowIncompatible=true`. -- Reading legacy INT96 timestamps contained within complex types can produce different results to Spark - There is a known performance issue when pushing filters down to Parquet. See the [Comet Tuning Guide] for more information. - There are failures in the Spark SQL test suite when enabling these new scans (tracking issues: [#1542] and [#1545]). diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala index 05901339b8..7f3aa24d2e 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala @@ -247,12 +247,7 @@ class CometFuzzTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("Parquet temporal types written as INT96") { - // int96 coercion in DF does not work with nested types yet - // https://github.com/apache/datafusion/issues/15763 - testParquetTemporalTypes( - ParquetOutputTimestampType.INT96, - generateArray = false, - generateStruct = false) + testParquetTemporalTypes(ParquetOutputTimestampType.INT96) } test("Parquet temporal types written as TIMESTAMP_MICROS") {