diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index e86d49ccefc2..1d52e57ab28d 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -101,18 +101,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with Logging { } override def supportsColumnarInput(schema: Seq[Attribute]): Boolean = { - // Note, there is a issue that, if gluten columnar scan is disabled and vanilla Spark - // columnar is enabled, then the following plan would fail. - // InMemoryTableScan - // InMemoryRelation - // (vanilla Spark columnar Scan) Parquet - // The reason is that, Spark will remove the top level `ColumnarToRow` and call - // `convertColumnarBatchToCachedBatch`, but the inside ColumnarBatch is not arrow-based. - // See: `InMemoryRelation.apply()`. - // So we should disallow columnar input if using vanilla Spark columnar scan. - val noVanillaSparkColumnarScan = glutenConf.enableColumnarFileScan || - !glutenConf.getConf(GlutenConfig.VANILLA_VECTORIZED_READERS_ENABLED) - glutenConf.enableGluten && validateSchema(schema) && noVanillaSparkColumnarScan + glutenConf.enableGluten && validateSchema(schema) } override def supportsColumnarOutput(schema: StructType): Boolean = {