From 4e01acc2d108d5463bb8319fb2a9df6b8f767746 Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Mon, 22 Sep 2025 14:53:17 +0800 Subject: [PATCH] fix: Do not push down subquery filters on native_datafusion scan --- .../org/apache/spark/sql/comet/CometScanExec.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala index e6804ca9f2..d4cb11ac62 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala @@ -156,7 +156,16 @@ case class CometScanExec( } @transient - private lazy val pushedDownFilters = getPushedDownFilters(relation, dataFilters) + private lazy val pushedDownFilters = { + val supportedFilters = if (scanImpl == CometConf.SCAN_NATIVE_DATAFUSION) { + // `native_datafusion` scan does not support subquery pushdown filters, + // see: https://github.com/apache/datafusion-comet/issues/2424 + dataFilters.filterNot(isDynamicPruningFilter) + } else { + dataFilters + } + getPushedDownFilters(relation, supportedFilters) + } override lazy val metadata: Map[String, String] = if (wrapped == null) Map.empty else wrapped.metadata