From c249b071d6607f53853383e616340a7ca3e3ec5e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 13 Oct 2022 14:45:34 -0400 Subject: [PATCH 1/2] Enable row pushdown and predicate reordering by default --- datafusion/core/src/config.rs | 6 +++--- .../core/src/physical_plan/file_format/parquet.rs | 12 ++++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/datafusion/core/src/config.rs b/datafusion/core/src/config.rs index 620634f259bac..c9dd56fbdd6d8 100644 --- a/datafusion/core/src/config.rs +++ b/datafusion/core/src/config.rs @@ -240,20 +240,20 @@ impl BuiltInConfigs { OPT_PARQUET_PUSHDOWN_FILTERS, "If true, filter expressions are be applied during the parquet decoding operation to \ reduce the number of rows decoded.", - false, + true, ), ConfigDefinition::new_bool( OPT_PARQUET_REORDER_FILTERS, "If true, filter expressions evaluated during the parquet decoding opearation \ will be reordered heuristically to minimize the cost of evaluation. If false, \ the filters are applied in the same order as written in the query.", - false, + true, ), ConfigDefinition::new_bool( OPT_PARQUET_ENABLE_PAGE_INDEX, "If true, uses parquet data page level metadata (Page Index) statistics \ to reduce the number of rows decoded.", - false, + true, ), ConfigDefinition::new_bool( OPT_OPTIMIZER_SKIP_FAILED_RULES, diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index 596d2b09e9506..e33346fa3b0c0 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -803,7 +803,7 @@ mod tests { let file_groups = meta.into_iter().map(Into::into).collect(); // prepare the scan - let mut parquet_exec = ParquetExec::new( + let parquet_exec = ParquetExec::new( FileScanConfig { object_store_url: ObjectStoreUrl::local_filesystem(), file_groups: vec![file_groups], @@ -817,13 +817,9 @@ mod tests { }, predicate, None, - ); - - if pushdown_predicate { - parquet_exec = parquet_exec - .with_pushdown_filters(true) - .with_reorder_filters(true); - } + ) + .with_pushdown_filters(pushdown_predicate) + .with_reorder_filters(pushdown_predicate); if page_index_predicate { parquet_exec = parquet_exec.with_enable_page_index(true); From 65ea1c218debca75b26cd7b1310c845c27827359 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 30 Nov 2022 14:23:14 -0500 Subject: [PATCH 2/2] fix compilation --- datafusion/core/src/physical_plan/file_format/parquet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index e33346fa3b0c0..99c7409a0e3bb 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -803,7 +803,7 @@ mod tests { let file_groups = meta.into_iter().map(Into::into).collect(); // prepare the scan - let parquet_exec = ParquetExec::new( + let mut parquet_exec = ParquetExec::new( FileScanConfig { object_store_url: ObjectStoreUrl::local_filesystem(), file_groups: vec![file_groups],