From 216163e6ce6d4437ea21a56bfbbae2ed717de647 Mon Sep 17 00:00:00 2001 From: jackwener Date: Mon, 6 Feb 2023 11:37:55 +0800 Subject: [PATCH 1/2] bugfix: fix `TableScan` may contain fields not included in `schema` --- datafusion/optimizer/src/push_down_projection.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datafusion/optimizer/src/push_down_projection.rs b/datafusion/optimizer/src/push_down_projection.rs index f3107d01b45f9..6b7d1a50c05ed 100644 --- a/datafusion/optimizer/src/push_down_projection.rs +++ b/datafusion/optimizer/src/push_down_projection.rs @@ -291,6 +291,10 @@ fn optimize_plan( // scans: // * remove un-used columns from the scan projection LogicalPlan::TableScan(scan) => { + // filter expr mayn't exist in expr in projection. + // like: TableScan: t1 projection=[bool_col, int_col], full_filters=[t1.id = Int32(1)] + // projection=[bool_col, int_col] don't contain `ti.id`. + exprlist_to_columns(&scan.filters, &mut new_required_columns)?; push_down_scan(scan, &new_required_columns, has_projection) } LogicalPlan::Explain { .. } => Err(DataFusionError::Internal( From ce62542308523c1c4eefd790cd03eccbc5c98a89 Mon Sep 17 00:00:00 2001 From: jakevin Date: Tue, 7 Feb 2023 14:51:57 +0800 Subject: [PATCH 2/2] polish comment Co-authored-by: Remzi Yang <59198230+HaoYang670@users.noreply.github.com> --- datafusion/optimizer/src/push_down_projection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/optimizer/src/push_down_projection.rs b/datafusion/optimizer/src/push_down_projection.rs index 6b7d1a50c05ed..f876542d96625 100644 --- a/datafusion/optimizer/src/push_down_projection.rs +++ b/datafusion/optimizer/src/push_down_projection.rs @@ -291,7 +291,7 @@ fn optimize_plan( // scans: // * remove un-used columns from the scan projection LogicalPlan::TableScan(scan) => { - // filter expr mayn't exist in expr in projection. + // filter expr may not exist in expr in projection. // like: TableScan: t1 projection=[bool_col, int_col], full_filters=[t1.id = Int32(1)] // projection=[bool_col, int_col] don't contain `ti.id`. exprlist_to_columns(&scan.filters, &mut new_required_columns)?;