From 916ddc31e8932c9653d99890ac26079a451762ec Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Tue, 13 Sep 2022 15:58:46 +0800 Subject: [PATCH] apply type coercion before filter pushdown --- datafusion/core/src/execution/context.rs | 4 +--- datafusion/core/tests/parquet_pruning.rs | 3 +-- datafusion/core/tests/sql/subqueries.rs | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs index 64f75dfa24de7..9540ca7ad9c4a 100644 --- a/datafusion/core/src/execution/context.rs +++ b/datafusion/core/src/execution/context.rs @@ -1448,10 +1448,8 @@ impl SessionState { rules.push(Arc::new(FilterNullJoinKeys::default())); } rules.push(Arc::new(ReduceOuterJoin::new())); - rules.push(Arc::new(FilterPushDown::new())); - // we do type coercion after filter push down so that we don't push CAST filters to Parquet - // until https://github.com/apache/arrow-datafusion/issues/3289 is resolved rules.push(Arc::new(TypeCoercion::new())); + rules.push(Arc::new(FilterPushDown::new())); rules.push(Arc::new(LimitPushDown::new())); rules.push(Arc::new(SingleDistinctToGroupBy::new())); diff --git a/datafusion/core/tests/parquet_pruning.rs b/datafusion/core/tests/parquet_pruning.rs index 6681a748ee790..1de1d7c8fe879 100644 --- a/datafusion/core/tests/parquet_pruning.rs +++ b/datafusion/core/tests/parquet_pruning.rs @@ -625,8 +625,7 @@ impl ContextWithParquet { .sql(sql) .await .expect("planning") - .to_logical_plan() - .unwrap(); + .to_unoptimized_plan(); self.run_test(logical_plan, sql).await } diff --git a/datafusion/core/tests/sql/subqueries.rs b/datafusion/core/tests/sql/subqueries.rs index 1ae5bc68e0d67..0ac286d76cd79 100644 --- a/datafusion/core/tests/sql/subqueries.rs +++ b/datafusion/core/tests/sql/subqueries.rs @@ -339,7 +339,7 @@ order by s_name; Projection: #lineitem.l_partkey, #lineitem.l_suppkey, Decimal128(Some(50000000000000000),38,17) * CAST(#SUM(lineitem.l_quantity) AS Decimal128(38, 17)) AS __value, alias=__sq_3 Aggregate: groupBy=[[#lineitem.l_partkey, #lineitem.l_suppkey]], aggr=[[SUM(#lineitem.l_quantity)]] Filter: #lineitem.l_shipdate >= Date32("8766") - TableScan: lineitem projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], partial_filters=[#lineitem.l_shipdate >= CAST(Utf8("1994-01-01") AS Date32)]"# + TableScan: lineitem projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], partial_filters=[#lineitem.l_shipdate >= Date32("8766")]"# .to_string(); assert_eq!(actual, expected); @@ -394,7 +394,7 @@ order by cntrycode;"#; Projection: #AVG(customer.c_acctbal) AS __value, alias=__sq_1 Aggregate: groupBy=[[]], aggr=[[AVG(#customer.c_acctbal)]] Filter: CAST(#customer.c_acctbal AS Decimal128(30, 15)) > Decimal128(Some(0),30,15) AND substr(#customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")]) - TableScan: customer projection=[c_phone, c_acctbal], partial_filters=[#customer.c_acctbal > Float64(0), substr(#customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])]"# + TableScan: customer projection=[c_phone, c_acctbal], partial_filters=[CAST(#customer.c_acctbal AS Decimal128(30, 15)) > Decimal128(Some(0),30,15), substr(#customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])]"# .to_string(); assert_eq!(actual, expected);