From 7c7ff9f10607d33899293c0aea905f8195bf0fef Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 20 Aug 2025 12:38:04 -0600 Subject: [PATCH 1/4] fall back to spark for schemas with empty structs --- spark/src/main/scala/org/apache/comet/DataTypeSupport.scala | 2 +- spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala | 2 ++ .../src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala index 66eab8c9f0..ab165a00d1 100644 --- a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala +++ b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala @@ -54,7 +54,7 @@ trait DataTypeSupport { TimestampNTZType => true case StructType(fields) => - fields.forall(f => isTypeSupported(f.dataType, f.name, fallbackReasons)) + fields.nonEmpty && fields.forall(f => isTypeSupported(f.dataType, f.name, fallbackReasons)) case ArrayType(elementType, _) => isTypeSupported(elementType, ARRAY_ELEMENT, fallbackReasons) case MapType(keyType, valueType, _) => diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 7ded26f23c..aad74064f6 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -357,6 +357,8 @@ case class CometScanTypeChecker(scanImpl: String) extends DataTypeSupport { false case _: StructType | _: ArrayType | _: MapType if scanImpl == CometConf.SCAN_NATIVE_COMET => false + case s: StructType if s.fields.isEmpty => + false case _ => super.isTypeSupported(dt, name, fallbackReasons) } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 1566e95e2b..cc1394677a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -197,7 +197,7 @@ object QueryPlanSerde extends Logging with CometExprShim { _: DoubleType | _: StringType | _: BinaryType | _: TimestampType | _: TimestampNTZType | _: DecimalType | _: DateType | _: BooleanType | _: NullType => true - case s: StructType if allowComplex => + case s: StructType if allowComplex && s.fields.nonEmpty => s.fields.map(_.dataType).forall(supportedDataType(_, allowComplex)) case a: ArrayType if allowComplex => supportedDataType(a.elementType, allowComplex) From bbf0e172934580c76ab5eb562a142b48e0fce47c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 20 Aug 2025 12:40:21 -0600 Subject: [PATCH 2/4] fall back to spark for schemas with empty structs --- .../main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index cc1394677a..1e95389c8c 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -197,8 +197,8 @@ object QueryPlanSerde extends Logging with CometExprShim { _: DoubleType | _: StringType | _: BinaryType | _: TimestampType | _: TimestampNTZType | _: DecimalType | _: DateType | _: BooleanType | _: NullType => true - case s: StructType if allowComplex && s.fields.nonEmpty => - s.fields.map(_.dataType).forall(supportedDataType(_, allowComplex)) + case s: StructType if allowComplex => + s.fields.nonEmpty && s.fields.map(_.dataType).forall(supportedDataType(_, allowComplex)) case a: ArrayType if allowComplex => supportedDataType(a.elementType, allowComplex) case m: MapType if allowComplex => From 44ffc29cf2c77af1e38a3f0b4cc38af50dd3f2f9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 20 Aug 2025 12:42:07 -0600 Subject: [PATCH 3/4] scalastyle --- spark/src/main/scala/org/apache/comet/DataTypeSupport.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala index ab165a00d1..9adf829580 100644 --- a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala +++ b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala @@ -54,7 +54,8 @@ trait DataTypeSupport { TimestampNTZType => true case StructType(fields) => - fields.nonEmpty && fields.forall(f => isTypeSupported(f.dataType, f.name, fallbackReasons)) + fields.nonEmpty && fields.forall(f => + isTypeSupported(f.dataType, f.name, fallbackReasons)) case ArrayType(elementType, _) => isTypeSupported(elementType, ARRAY_ELEMENT, fallbackReasons) case MapType(keyType, valueType, _) => From f6b22c8c2503d26cdbbfca285e8d6bfa690ffcea Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 20 Aug 2025 13:10:02 -0600 Subject: [PATCH 4/4] trigger CI