From 6b371cb2c0fc7ac23026eb1104e1a1df5e50947c Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 24 Apr 2025 11:11:18 +0100 Subject: [PATCH] fixup --- .../execution/ColumnarCachedBatchSerializer.scala | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index e86d49ccefc2..1d52e57ab28d 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -101,18 +101,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with Logging { } override def supportsColumnarInput(schema: Seq[Attribute]): Boolean = { - // Note, there is a issue that, if gluten columnar scan is disabled and vanilla Spark - // columnar is enabled, then the following plan would fail. - // InMemoryTableScan - // InMemoryRelation - // (vanilla Spark columnar Scan) Parquet - // The reason is that, Spark will remove the top level `ColumnarToRow` and call - // `convertColumnarBatchToCachedBatch`, but the inside ColumnarBatch is not arrow-based. - // See: `InMemoryRelation.apply()`. - // So we should disallow columnar input if using vanilla Spark columnar scan. - val noVanillaSparkColumnarScan = glutenConf.enableColumnarFileScan || - !glutenConf.getConf(GlutenConfig.VANILLA_VECTORIZED_READERS_ENABLED) - glutenConf.enableGluten && validateSchema(schema) && noVanillaSparkColumnarScan + glutenConf.enableGluten && validateSchema(schema) } override def supportsColumnarOutput(schema: StructType): Boolean = {