From 204a48166cb2e3faf135298cee60c2fa31b35965 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 12 Aug 2024 21:18:02 +0800 Subject: [PATCH 1/9] [Feat](nereids) add max/min filter push down rewrite rule --- .../doris/nereids/jobs/executor/Rewriter.java | 4 +- .../apache/doris/nereids/rules/RuleType.java | 1 + .../rules/rewrite/MaxMinFilterPushDown.java | 129 ++++++++++++++++++ .../max_min_filter_push_down.out | 127 +++++++++++++++++ .../max_min_filter_push_down.groovy | 102 ++++++++++++++ 5 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java create mode 100644 regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out create mode 100644 regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 715463b496b3bc..e62b1cd0d59869 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -87,6 +87,7 @@ import org.apache.doris.nereids.rules.rewrite.LimitAggToTopNAgg; import org.apache.doris.nereids.rules.rewrite.LimitSortToTopN; import org.apache.doris.nereids.rules.rewrite.LogicalResultSinkToShortCircuitPointQuery; +import org.apache.doris.nereids.rules.rewrite.MaxMinFilterPushDown; import org.apache.doris.nereids.rules.rewrite.MergeAggregate; import org.apache.doris.nereids.rules.rewrite.MergeFilters; import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion; @@ -181,7 +182,8 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown( // ExtractSingleTableExpressionFromDisjunction conflict to InPredicateToEqualToRule // in the ExpressionNormalization, so must invoke in another job, otherwise dead loop. - new ExtractSingleTableExpressionFromDisjunction() + new ExtractSingleTableExpressionFromDisjunction(), + new MaxMinFilterPushDown() ) ), // subquery unnesting relay on ExpressionNormalization to extract common factor expression diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index f2c572f7779e91..ca26ab1d9f843c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -176,6 +176,7 @@ public enum RuleType { PUSH_DOWN_FILTER_THROUGH_CTE(RuleTypeClass.REWRITE), PUSH_DOWN_FILTER_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE), + MAX_MIN_FILTER_PUSH_DOWN(RuleTypeClass.REWRITE), PUSH_DOWN_DISTINCT_THROUGH_JOIN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java new file mode 100644 index 00000000000000..45eeaef9025bb2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * select id, max(a) from t group by id having max(a)>10; + * -> + * select id, max(a) from t where a>10 group by id; + * select id, min(a) from t group by id having min(a)<10; + * -> + * select id, min(a) from t where a<10 group by id; + */ +@DependsRules({ + ExpressionRewrite.class +}) +public class MaxMinFilterPushDown extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalFilter(logicalAggregate()) + .then(this::pushDownMaxMinFilter) + .toRule(RuleType.MAX_MIN_FILTER_PUSH_DOWN); + } + + private Plan pushDownMaxMinFilter(LogicalFilter> filter) { + Set conjuncts = filter.getConjuncts(); + LogicalAggregate agg = filter.child(); + Plan aggChild = agg.child(); + List aggOutputExpressions = agg.getOutputExpressions(); + Set maxMinFunc = ExpressionUtils.collect(aggOutputExpressions, + expr -> expr instanceof Max || expr instanceof Min); + // LogicalAggregate only outputs one aggregate function, which is max or min + if (maxMinFunc.size() != 1) { + return null; + } + ExprId exprId = null; + Expression func = maxMinFunc.iterator().next(); + for (NamedExpression expr : aggOutputExpressions) { + if (expr instanceof Alias && ((Alias) expr).child().equals(func)) { + Alias alias = (Alias) expr; + exprId = alias.getExprId(); + } + } + // try to find min(a)<10 or max(a)>10 + Expression originConjunct = findMatchingConjunct(conjuncts, func instanceof Max, exprId).orElse(null); + if (null == originConjunct) { + return null; + } + Set newUpperConjuncts = new HashSet<>(conjuncts); + newUpperConjuncts.remove(originConjunct); + Expression newPredicate = null; + if (func instanceof Max) { + if (originConjunct instanceof GreaterThan) { + newPredicate = new GreaterThan(func.child(0), originConjunct.child(1)); + } else if (originConjunct instanceof GreaterThanEqual) { + newPredicate = new GreaterThanEqual(func.child(0), originConjunct.child(1)); + } + } else { + if (originConjunct instanceof LessThan) { + newPredicate = new LessThan(func.child(0), originConjunct.child(1)); + } else if (originConjunct instanceof LessThanEqual) { + newPredicate = new LessThanEqual(func.child(0), originConjunct.child(1)); + } + } + + LogicalFilter newPushDownFilter = new LogicalFilter<>(ImmutableSet.of(newPredicate), aggChild); + LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newPushDownFilter)); + return PlanUtils.filterOrSelf(newUpperConjuncts, newAgg); + } + + private Optional findMatchingConjunct(Set conjuncts, boolean isMax, ExprId exprId) { + for (Expression conjunct : conjuncts) { + if ((isMax && (conjunct instanceof GreaterThan || conjunct instanceof GreaterThanEqual)) + || (!isMax && (conjunct instanceof LessThan || conjunct instanceof LessThanEqual))) { + if (conjunct.child(0) instanceof SlotReference && conjunct.child(1) instanceof Literal) { + SlotReference slot = (SlotReference) conjunct.child(0); + if (slot.getExprId().equals(exprId)) { + return Optional.of(conjunct); + } + } + } + } + return Optional.empty(); + } +} diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out new file mode 100644 index 00000000000000..5c450b9267969d --- /dev/null +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -0,0 +1,127 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !min -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 20)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_commute -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 <= 20)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 >= 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !min_commute_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 <= 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max_commute_equal -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 >= 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !has_other_agg_func -- +PhysicalResultSink +--filter((max(value1) >= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !smallint -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_smallint > 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !tinyint -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_tinyint < 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !char100 -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_char100 > 'ab')) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !char100_cmp_num_cannot_rewrite -- +PhysicalResultSink +--filter((cast(min(d_char100) as DOUBLE) < 10.0)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !datetimev2 -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_datetimev2 < '2020-01-09 00:00:00')) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !datev2 -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_datev2 > '2020-01-09')) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !smallint_group_by_key -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_smallint > 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !tinyint_group_by_key -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_tinyint < 10)) +--------PhysicalOlapScan[max_min_filter_push_down2] + +-- !char100_group_by_key -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_char100 > 'ab')) +--------PhysicalOlapScan[max_min_filter_push_down2] + diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy new file mode 100644 index 00000000000000..01ed392d815550 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("max_min_filter_push_down") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql "drop table if exists max_min_filter_push_down1" + sql""" + CREATE TABLE max_min_filter_push_down1 ( + id INT, + value1 INT, + value2 VARCHAR(50) + ); + """ + + sql """ + INSERT INTO max_min_filter_push_down1 (id, value1, value2) VALUES + (1, 10, 'A'), + (2, 20, 'B'), + (3, 30, 'C'), + (4, 40, 'D'); + """ + + qt_min """ + explain shape plan + select id,min(value1) from max_min_filter_push_down1 group by id having min(value1) <40 and min(value1) <20; + """ + qt_max """ + explain shape plan + select id,max(value1) from max_min_filter_push_down1 group by id having max(value1) >40; + """ + + qt_min_commute """ + explain shape plan + select id,min(value1) from max_min_filter_push_down1 group by id having 40>min(value1); + """ + qt_max """ + explain shape plan + select id,max(value1) from max_min_filter_push_down1 group by id having 40=40; + """ + + qt_min_commute_equal """ + explain shape plan + select id,min(value1) from max_min_filter_push_down1 group by id having 40>=min(value1); + """ + qt_max_commute_equal """ + explain shape plan + select id,max(value1) from max_min_filter_push_down1 group by id having 40<=max(value1); + """ + + qt_has_other_agg_func """ + explain shape plan + select id,max(value1),min(value1) from max_min_filter_push_down1 group by id having 40<=max(value1); + """ + + sql "drop table if exists max_min_filter_push_down2" + sql "create table max_min_filter_push_down2(d_int int, d_char100 char(100), d_smallint smallint, d_tinyint tinyint, d_char10 char(10),d_datetimev2 datetimev2, d_datev2 datev2) ;" + sql "insert into max_min_filter_push_down2 values(1,'01234567890123456789', 3,3,'0123456789','2020-01-09 10:00:00.99','2020-01-09'),(14,'01234567890123456789', 33,23,'0123456789','2020-01-11 10:00:00.99','2020-01-11');" + + qt_smallint """explain shape plan + select d_int,max(d_smallint) from max_min_filter_push_down2 group by d_int having max(d_smallint)>10;""" + qt_tinyint """explain shape plan + select d_int,min(d_tinyint) from max_min_filter_push_down2 group by d_int having min(d_tinyint)<10;""" + qt_char100 """explain shape plan + select d_int,max(d_char100) from max_min_filter_push_down2 group by d_int having max(d_char100)>'ab';""" + qt_char100_cmp_num_cannot_rewrite """explain shape plan + select d_int,min(d_char100) from max_min_filter_push_down2 group by d_int having min(d_char100)<10;""" + qt_datetimev2 """explain shape plan + select d_int,min(d_datetimev2) from max_min_filter_push_down2 group by d_int having min(d_datetimev2)<'2020-01-09';""" + qt_datev2 """explain shape plan + select d_int,max(d_datev2) from max_min_filter_push_down2 group by d_int having max(d_datev2)>'2020-01-09 10:00:00';""" + qt_smallint_group_by_key """explain shape plan + select max(d_smallint) from max_min_filter_push_down2 group by d_smallint having max(d_smallint)>10;""" + qt_tinyint_group_by_key """explain shape plan + select min(d_tinyint) from max_min_filter_push_down2 group by d_tinyint having min(d_tinyint)<10;""" + qt_char100_group_by_key """explain shape plan + select max(d_char100) from max_min_filter_push_down2 group by d_char100 having max(d_char100)>'ab';""" +} \ No newline at end of file From b7a3cbc986d9216f6e2236f103ab1335ae9bd8bd Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 13 Aug 2024 16:29:18 +0800 Subject: [PATCH 2/9] [Feat](nereids) add max/min filter push down rewrite rule --- .../rules/rewrite/MaxMinFilterPushDown.java | 7 ++- .../rewrite/MaxMinFilterPushDownTest.java | 48 +++++++++++++++++++ .../push_down_max_through_join.out | 20 ++++---- .../max_min_filter_push_down.groovy | 5 +- 4 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java index 45eeaef9025bb2..8e7020a8d01b50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.LessThanEqual; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -71,10 +72,12 @@ private Plan pushDownMaxMinFilter(LogicalFilter> filter) LogicalAggregate agg = filter.child(); Plan aggChild = agg.child(); List aggOutputExpressions = agg.getOutputExpressions(); - Set maxMinFunc = ExpressionUtils.collect(aggOutputExpressions, + Set aggFuncs = ExpressionUtils.collect(aggOutputExpressions, + expr -> expr instanceof AggregateFunction); + Set maxMinFunc = ExpressionUtils.collect(aggFuncs, expr -> expr instanceof Max || expr instanceof Min); // LogicalAggregate only outputs one aggregate function, which is max or min - if (maxMinFunc.size() != 1) { + if (aggFuncs.size() != 1 || maxMinFunc.size() != 1) { return null; } ExprId exprId = null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java new file mode 100644 index 00000000000000..72650c7a396880 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java @@ -0,0 +1,48 @@ +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Test; + + +public class MaxMinFilterPushDownTest extends TestWithFeService implements MemoPatternMatchSupported { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + connectContext.setDatabase("test"); + createTable("CREATE TABLE IF NOT EXISTS max_t(\n" + + "`id` int(32),\n" + + "`score` int(64) NULL,\n" + + "`name` varchar(64) NULL\n" + + ") properties('replication_num'='1');"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + } + + @Test + public void testMaxRewrite() { + String sql = "select id, max(score) from max_t group by id having max(score)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getConjuncts().size() == 1)); + } + @Test + public void testMinRewrite() { + String sql = "select id, min(score) from max_t group by id having min(score)<10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getConjuncts().size() == 1)); + } + + @Test + public void testMaxNotRewrite() { + String sql = "select id, max(score) from max_t group by id having max(score)<10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + @Test + public void testMinNotRewrite() { + String sql = "select id, min(score) from max_t group by id having min(score)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } +} diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out index 281de8ea61b88e..79b4ed890ded95 100644 --- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out +++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_max_through_join.out @@ -91,11 +91,11 @@ PhysicalResultSink -- !groupby_pushdown_having -- PhysicalResultSink ---filter((max(score) > 100)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() -----------PhysicalOlapScan[max_t] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() +--------PhysicalOlapScan[max_t] +--------filter((t1.score > 100)) ----------PhysicalOlapScan[max_t] -- !groupby_pushdown_mixed_aggregates -- @@ -366,11 +366,11 @@ SyntaxError: -- !with_hint_groupby_pushdown_having -- PhysicalResultSink ---filter((max(score) > 100)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() -----------PhysicalOlapScan[max_t] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() +--------PhysicalOlapScan[max_t] +--------filter((t1.score > 100)) ----------PhysicalOlapScan[max_t] Hint log: diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy index 01ed392d815550..ce71d4c4e06a59 100644 --- a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -25,7 +25,7 @@ suite("max_min_filter_push_down") { id INT, value1 INT, value2 VARCHAR(50) - ); + ) properties("replication_num"="1"); """ sql """ @@ -78,7 +78,8 @@ suite("max_min_filter_push_down") { """ sql "drop table if exists max_min_filter_push_down2" - sql "create table max_min_filter_push_down2(d_int int, d_char100 char(100), d_smallint smallint, d_tinyint tinyint, d_char10 char(10),d_datetimev2 datetimev2, d_datev2 datev2) ;" + sql """create table max_min_filter_push_down2(d_int int, d_char100 char(100), d_smallint smallint, d_tinyint tinyint, d_char10 char(10),d_datetimev2 datetimev2, d_datev2 datev2) + properties("replication_num"="1");""" sql "insert into max_min_filter_push_down2 values(1,'01234567890123456789', 3,3,'0123456789','2020-01-09 10:00:00.99','2020-01-09'),(14,'01234567890123456789', 33,23,'0123456789','2020-01-11 10:00:00.99','2020-01-11');" qt_smallint """explain shape plan From cd9035da20254bf56b5a4850d9b2e59e60148155 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 13 Aug 2024 21:35:17 +0800 Subject: [PATCH 3/9] [Feat](nereids) add max/min filter push down rewrite rule --- .../rules/rewrite/MaxMinFilterPushDown.java | 3 +- .../rewrite/MaxMinFilterPushDownTest.java | 22 +- .../max_min_filter_push_down.out | 217 ++++++++++++++---- .../max_min_filter_push_down.groovy | 92 +++++++- 4 files changed, 278 insertions(+), 56 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java index 8e7020a8d01b50..c137a5ffa3a97b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java @@ -40,6 +40,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -109,7 +110,7 @@ private Plan pushDownMaxMinFilter(LogicalFilter> filter) newPredicate = new LessThanEqual(func.child(0), originConjunct.child(1)); } } - + Preconditions.checkState(newPredicate != null); LogicalFilter newPushDownFilter = new LogicalFilter<>(ImmutableSet.of(newPredicate), aggChild); LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newPushDownFilter)); return PlanUtils.filterOrSelf(newUpperConjuncts, newAgg); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java index 72650c7a396880..d4461162424926 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java @@ -34,15 +34,33 @@ public void testMinRewrite() { } @Test - public void testMaxNotRewrite() { + public void testMaxNotRewrite0() { String sql = "select id, max(score) from max_t group by id having max(score)<10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } @Test - public void testMinNotRewrite() { + public void testMinNotRewrite1() { String sql = "select id, min(score) from max_t group by id having min(score)>10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } + @Test + public void testMinNotRewrite2() { + String sql = "select id, min(score), max(score) from max_t group by id having min(score)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + @Test + public void testMinNotRewrite3() { + String sql = "select id, min(score), count(score) from max_t group by id having min(score)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + @Test + public void testMinNotRewrite4() { + String sql = "select id, max(score) from max_t group by id having abs(max(score))>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } } diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out index 5c450b9267969d..9439ddb120a11f 100644 --- a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -1,58 +1,58 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !min -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 < 20)) +--filter((min(value1) < 20)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !max -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 > 40)) +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !min_commute -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 < 40)) +--filter((min(value1) < 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !max -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 > 40)) +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !min_equal -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 <= 20)) +--filter((min(value1) <= 20)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !max_equal -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 >= 40)) +--filter((max(value1) >= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !min_commute_equal -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 <= 40)) +--filter((min(value1) <= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !max_commute_equal -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 >= 40)) +--filter((max(value1) >= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] -- !has_other_agg_func -- @@ -62,25 +62,119 @@ PhysicalResultSink ------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down1] +-- !min_scalar_agg -- +PhysicalResultSink +--filter((min(value1) < 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !max_scalar_agg -- +PhysicalResultSink +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !max_scalar_agg -- +PhysicalResultSink +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !min_equal_scalar_agg -- +PhysicalResultSink +--filter((min(value1) <= 20)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !max_equal_scalar_agg -- +PhysicalResultSink +--filter((max(value1) >= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !min_res -- +1 10 +2 19 + +-- !max_res -- +2 73 +3 61 +4 45 + +-- !min_commute_res -- +1 10 +2 19 +3 30 + +-- !max_res -- +2 73 +3 61 +4 45 + +-- !min_equal_res -- +1 10 +2 19 + +-- !max_equal_res -- +2 73 +3 61 +4 45 + +-- !min_commute_equal_res -- +1 10 +2 19 +3 30 +4 40 + +-- !max_commute_equal_res -- +2 73 +3 61 +4 45 + +-- !has_other_agg_func_res -- +2 73 19 +3 61 30 +4 45 40 + +-- !min_scalar_agg_res -- +10 + +-- !max_scalar_agg_res -- +73 + +-- !max_scalar_agg_res -- +73 + +-- !min_equal_scalar_agg_res -- +10 + +-- !max_equal_scalar_agg_res -- +73 + -- !smallint -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_smallint > 10)) +--filter((max(d_smallint) > 10)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] -- !tinyint -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_tinyint < 10)) +--filter((min(d_tinyint) < 10)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] -- !char100 -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_char100 > 'ab')) +--filter((max(d_char100) > 'ab')) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] -- !char100_cmp_num_cannot_rewrite -- @@ -92,36 +186,63 @@ PhysicalResultSink -- !datetimev2 -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_datetimev2 < '2020-01-09 00:00:00')) +--filter((min(d_datetimev2) < '2020-01-09 00:00:00')) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] -- !datev2 -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_datev2 > '2020-01-09')) +--filter((max(d_datev2) > '2020-01-09')) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] -- !smallint_group_by_key -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_smallint > 10)) +--filter((max(d_smallint) > 10)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] -- !tinyint_group_by_key -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_tinyint < 10)) +--filter((min(d_tinyint) < 10)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] -- !char100_group_by_key -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down2.d_char100 > 'ab')) +--filter((max(d_char100) > 'ab')) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] --------PhysicalOlapScan[max_min_filter_push_down2] +-- !smallint_res -- +14 32 + +-- !tinyint_res -- +1 3 + +-- !char100_res -- + +-- !char100_cmp_num_cannot_rewrite_res -- + +-- !datetimev2_res -- +1 2020-01-07T10:00:01 +14 2020-01-07T10:00:01 + +-- !datev2_res -- +1 2020-01-11 +14 2020-01-11 + +-- !smallint_group_by_key_res -- +29 +32 + +-- !tinyint_group_by_key_res -- +3 + +-- !char100_group_by_key_res -- + diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy index ce71d4c4e06a59..9beb80b414b013 100644 --- a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -30,10 +30,7 @@ suite("max_min_filter_push_down") { sql """ INSERT INTO max_min_filter_push_down1 (id, value1, value2) VALUES - (1, 10, 'A'), - (2, 20, 'B'), - (3, 30, 'C'), - (4, 40, 'D'); + (1, 10, 'A'),(1, 11, 'A'),(2, 20, 'B'),(2, 73, 'B'),(2, 19, 'B'),(3, 30, 'C'),(3, 61, 'C'),(4, 40, 'D'),(4, 43, 'D'),(4, 45, 'D'); """ qt_min """ @@ -77,10 +74,85 @@ suite("max_min_filter_push_down") { select id,max(value1),min(value1) from max_min_filter_push_down1 group by id having 40<=max(value1); """ + qt_min_scalar_agg """ + explain shape plan + select min(value1) from max_min_filter_push_down1 having min(value1) <40; + """ + qt_max_scalar_agg """ + explain shape plan + select max(value1) from max_min_filter_push_down1 having max(value1) >40; + """ + qt_max_scalar_agg """ + explain shape plan + select max(value1) from max_min_filter_push_down1 having 40=40; + """ + + + qt_min_res """ + select id,min(value1) from max_min_filter_push_down1 group by id having min(value1) <40 and min(value1) <20 order by 1,2; + """ + qt_max_res """ + select id,max(value1) from max_min_filter_push_down1 group by id having max(value1) >40 order by 1,2; + """ + + qt_min_commute_res """ + select id,min(value1) from max_min_filter_push_down1 group by id having 40>min(value1) order by 1,2; + """ + qt_max_res """ + select id,max(value1) from max_min_filter_push_down1 group by id having 40=40 order by 1,2; + """ + + qt_min_commute_equal_res """ + select id,min(value1) from max_min_filter_push_down1 group by id having 40>=min(value1) order by 1,2; + """ + qt_max_commute_equal_res """ + select id,max(value1) from max_min_filter_push_down1 group by id having 40<=max(value1) order by 1,2; + """ + + qt_has_other_agg_func_res """ + select id,max(value1),min(value1) from max_min_filter_push_down1 group by id having 40<=max(value1) order by 1,2; + """ + + qt_min_scalar_agg_res """ + select min(value1) from max_min_filter_push_down1 having min(value1) <40; + """ + qt_max_scalar_agg_res """ + select max(value1) from max_min_filter_push_down1 having max(value1) >40; + """ + qt_max_scalar_agg_res """ + select max(value1) from max_min_filter_push_down1 having 40=40; + """ + + sql "drop table if exists max_min_filter_push_down2" sql """create table max_min_filter_push_down2(d_int int, d_char100 char(100), d_smallint smallint, d_tinyint tinyint, d_char10 char(10),d_datetimev2 datetimev2, d_datev2 datev2) properties("replication_num"="1");""" - sql "insert into max_min_filter_push_down2 values(1,'01234567890123456789', 3,3,'0123456789','2020-01-09 10:00:00.99','2020-01-09'),(14,'01234567890123456789', 33,23,'0123456789','2020-01-11 10:00:00.99','2020-01-11');" + sql """insert into max_min_filter_push_down2 values(1,'01234567890123456789', 3,3,'0123456789','2020-01-09 10:00:00.99','2020-01-09') + ,(14,'01234567890123456789', 29,23,'0123456789','2020-01-7 10:00:00.99','2020-01-11'),(1,'01234567890123456789', 7,23,'0123456789','2020-01-7 10:00:00.99','2020-01-11') + ,(14,'01234567890123456789', 32,23,'0123456789','2020-01-11 10:00:00.99','2020-01-11'),(1,'01234567890123456789', 8,23,'0123456789','2020-01-11 10:00:00.99','2020-01-11');""" qt_smallint """explain shape plan select d_int,max(d_smallint) from max_min_filter_push_down2 group by d_int having max(d_smallint)>10;""" @@ -100,4 +172,14 @@ suite("max_min_filter_push_down") { select min(d_tinyint) from max_min_filter_push_down2 group by d_tinyint having min(d_tinyint)<10;""" qt_char100_group_by_key """explain shape plan select max(d_char100) from max_min_filter_push_down2 group by d_char100 having max(d_char100)>'ab';""" + + qt_smallint_res """select d_int,max(d_smallint) from max_min_filter_push_down2 group by d_int having max(d_smallint)>10 order by 1,2;""" + qt_tinyint_res """select d_int,min(d_tinyint) from max_min_filter_push_down2 group by d_int having min(d_tinyint)<10 order by 1,2;""" + qt_char100_res """select d_int,max(d_char100) from max_min_filter_push_down2 group by d_int having max(d_char100)>'ab' order by 1,2;""" + qt_char100_cmp_num_cannot_rewrite_res """select d_int,min(d_char100) from max_min_filter_push_down2 group by d_int having min(d_char100)<10 order by 1,2;""" + qt_datetimev2_res """select d_int,min(d_datetimev2) from max_min_filter_push_down2 group by d_int having min(d_datetimev2)<'2020-01-09' order by 1,2;""" + qt_datev2_res """select d_int,max(d_datev2) from max_min_filter_push_down2 group by d_int having max(d_datev2)>'2020-01-09 10:00:00' order by 1,2;""" + qt_smallint_group_by_key_res """select max(d_smallint) from max_min_filter_push_down2 group by d_smallint having max(d_smallint)>10 order by 1,2;""" + qt_tinyint_group_by_key_res """select min(d_tinyint) from max_min_filter_push_down2 group by d_tinyint having min(d_tinyint)<10 order by 1,2;""" + qt_char100_group_by_key_res """select max(d_char100) from max_min_filter_push_down2 group by d_char100 having max(d_char100)>'ab' order by 1,2;""" } \ No newline at end of file From 7a84236efa1cf534294d3136d40e581c59db4a4b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 14 Aug 2024 10:48:09 +0800 Subject: [PATCH 4/9] [Feat](nereids) add max/min filter push down rewrite rule --- .../rewrite/MaxMinFilterPushDownTest.java | 23 ++++++++++++++++++- .../max_min_filter_push_down.groovy | 6 ++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java index d4461162424926..0c8cced405d317 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.util.MemoPatternMatchSupported; @@ -6,7 +23,6 @@ import org.junit.jupiter.api.Test; - public class MaxMinFilterPushDownTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { @@ -26,6 +42,7 @@ public void testMaxRewrite() { PlanChecker.from(connectContext).analyze(sql).rewrite() .matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getConjuncts().size() == 1)); } + @Test public void testMinRewrite() { String sql = "select id, min(score) from max_t group by id having min(score)<10"; @@ -39,24 +56,28 @@ public void testMaxNotRewrite0() { PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } + @Test public void testMinNotRewrite1() { String sql = "select id, min(score) from max_t group by id having min(score)>10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } + @Test public void testMinNotRewrite2() { String sql = "select id, min(score), max(score) from max_t group by id having min(score)>10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } + @Test public void testMinNotRewrite3() { String sql = "select id, min(score), count(score) from max_t group by id having min(score)>10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } + @Test public void testMinNotRewrite4() { String sql = "select id, max(score) from max_t group by id having abs(max(score))>10"; diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy index 9beb80b414b013..98c48570dfdb55 100644 --- a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -179,7 +179,7 @@ suite("max_min_filter_push_down") { qt_char100_cmp_num_cannot_rewrite_res """select d_int,min(d_char100) from max_min_filter_push_down2 group by d_int having min(d_char100)<10 order by 1,2;""" qt_datetimev2_res """select d_int,min(d_datetimev2) from max_min_filter_push_down2 group by d_int having min(d_datetimev2)<'2020-01-09' order by 1,2;""" qt_datev2_res """select d_int,max(d_datev2) from max_min_filter_push_down2 group by d_int having max(d_datev2)>'2020-01-09 10:00:00' order by 1,2;""" - qt_smallint_group_by_key_res """select max(d_smallint) from max_min_filter_push_down2 group by d_smallint having max(d_smallint)>10 order by 1,2;""" - qt_tinyint_group_by_key_res """select min(d_tinyint) from max_min_filter_push_down2 group by d_tinyint having min(d_tinyint)<10 order by 1,2;""" - qt_char100_group_by_key_res """select max(d_char100) from max_min_filter_push_down2 group by d_char100 having max(d_char100)>'ab' order by 1,2;""" + qt_smallint_group_by_key_res """select max(d_smallint) from max_min_filter_push_down2 group by d_smallint having max(d_smallint)>10 order by 1;""" + qt_tinyint_group_by_key_res """select min(d_tinyint) from max_min_filter_push_down2 group by d_tinyint having min(d_tinyint)<10 order by 1;""" + qt_char100_group_by_key_res """select max(d_char100) from max_min_filter_push_down2 group by d_char100 having max(d_char100)>'ab' order by 1;""" } \ No newline at end of file From d93e54e7af5428069e82937103bc1f56411d5479 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 15 Aug 2024 10:55:22 +0800 Subject: [PATCH 5/9] fix regression --- .../max_min_filter_push_down.out | 136 +++++++++--------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out index 9439ddb120a11f..ff463078936b21 100644 --- a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -1,58 +1,58 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !min -- PhysicalResultSink ---filter((min(value1) < 20)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 20)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !max -- PhysicalResultSink ---filter((max(value1) > 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !min_commute -- PhysicalResultSink ---filter((min(value1) < 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 40)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !max -- PhysicalResultSink ---filter((max(value1) > 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !min_equal -- PhysicalResultSink ---filter((min(value1) <= 20)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 <= 20)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !max_equal -- PhysicalResultSink ---filter((max(value1) >= 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 >= 40)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !min_commute_equal -- PhysicalResultSink ---filter((min(value1) <= 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 <= 40)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !max_commute_equal -- PhysicalResultSink ---filter((max(value1) >= 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 >= 40)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !has_other_agg_func -- @@ -64,38 +64,38 @@ PhysicalResultSink -- !min_scalar_agg -- PhysicalResultSink ---filter((min(value1) < 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------PhysicalStorageLayerAggregate[max_min_filter_push_down1] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] -- !max_scalar_agg -- PhysicalResultSink ---filter((max(value1) > 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------PhysicalStorageLayerAggregate[max_min_filter_push_down1] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] -- !max_scalar_agg -- PhysicalResultSink ---filter((max(value1) > 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------PhysicalStorageLayerAggregate[max_min_filter_push_down1] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 > 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] -- !min_equal_scalar_agg -- PhysicalResultSink ---filter((min(value1) <= 20)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------PhysicalStorageLayerAggregate[max_min_filter_push_down1] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 <= 20)) +--------PhysicalOlapScan[max_min_filter_push_down1] -- !max_equal_scalar_agg -- PhysicalResultSink ---filter((max(value1) >= 40)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] ---------PhysicalStorageLayerAggregate[max_min_filter_push_down1] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 >= 40)) +--------PhysicalOlapScan[max_min_filter_push_down1] -- !min_res -- 1 10 @@ -158,23 +158,23 @@ PhysicalResultSink -- !smallint -- PhysicalResultSink ---filter((max(d_smallint) > 10)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_smallint > 10)) --------PhysicalOlapScan[max_min_filter_push_down2] -- !tinyint -- PhysicalResultSink ---filter((min(d_tinyint) < 10)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_tinyint < 10)) --------PhysicalOlapScan[max_min_filter_push_down2] -- !char100 -- PhysicalResultSink ---filter((max(d_char100) > 'ab')) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_char100 > 'ab')) --------PhysicalOlapScan[max_min_filter_push_down2] -- !char100_cmp_num_cannot_rewrite -- @@ -186,37 +186,37 @@ PhysicalResultSink -- !datetimev2 -- PhysicalResultSink ---filter((min(d_datetimev2) < '2020-01-09 00:00:00')) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_datetimev2 < '2020-01-09 00:00:00')) --------PhysicalOlapScan[max_min_filter_push_down2] -- !datev2 -- PhysicalResultSink ---filter((max(d_datev2) > '2020-01-09')) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_datev2 > '2020-01-09')) --------PhysicalOlapScan[max_min_filter_push_down2] -- !smallint_group_by_key -- PhysicalResultSink ---filter((max(d_smallint) > 10)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_smallint > 10)) --------PhysicalOlapScan[max_min_filter_push_down2] -- !tinyint_group_by_key -- PhysicalResultSink ---filter((min(d_tinyint) < 10)) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_tinyint < 10)) --------PhysicalOlapScan[max_min_filter_push_down2] -- !char100_group_by_key -- PhysicalResultSink ---filter((max(d_char100) > 'ab')) -----hashAgg[GLOBAL] -------hashAgg[LOCAL] +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down2.d_char100 > 'ab')) --------PhysicalOlapScan[max_min_filter_push_down2] -- !smallint_res -- From ee3edfc7768a45b3a76fd6843755c833bbae205e Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 16 Aug 2024 18:03:29 +0800 Subject: [PATCH 6/9] add regression --- .../max_min_filter_push_down.out | 22 +++++++++++++++++++ .../max_min_filter_push_down.groovy | 16 +++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out index ff463078936b21..4ad1baf3387200 100644 --- a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -13,6 +13,20 @@ PhysicalResultSink ------filter((max_min_filter_push_down1.value1 > 40)) --------PhysicalOlapScan[max_min_filter_push_down1] +-- !min_expr -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((cast(value1 as BIGINT) < 19)) +--------PhysicalOlapScan[max_min_filter_push_down1] + +-- !max_expr -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((abs(value1) > 39)) +--------PhysicalOlapScan[max_min_filter_push_down1] + -- !min_commute -- PhysicalResultSink --hashAgg[GLOBAL] @@ -106,6 +120,14 @@ PhysicalResultSink 3 61 4 45 +-- !min_expr_res -- +1 11 + +-- !max_expr_res -- +2 74 +3 62 +4 46 + -- !min_commute_res -- 1 10 2 19 diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy index 98c48570dfdb55..b1acd02d0e5c6d 100644 --- a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -42,6 +42,15 @@ suite("max_min_filter_push_down") { select id,max(value1) from max_min_filter_push_down1 group by id having max(value1) >40; """ + qt_min_expr """ + explain shape plan + select id,min(value1+1) from max_min_filter_push_down1 group by id having min(value1+1) <40 and min(value1+1) <20; + """ + qt_max_expr """ + explain shape plan + select id,max(abs(value1)+1) from max_min_filter_push_down1 group by id having max(abs(value1)+1) >40; + """ + qt_min_commute """ explain shape plan select id,min(value1) from max_min_filter_push_down1 group by id having 40>min(value1); @@ -103,7 +112,12 @@ suite("max_min_filter_push_down") { qt_max_res """ select id,max(value1) from max_min_filter_push_down1 group by id having max(value1) >40 order by 1,2; """ - + qt_min_expr_res """ + select id,min(value1+1) from max_min_filter_push_down1 group by id having min(value1+1) <40 and min(value1+1) <20 order by 1,2; + """ + qt_max_expr_res """ + select id,max(abs(value1)+1) from max_min_filter_push_down1 group by id having max(abs(value1)+1) >40 order by 1,2; + """ qt_min_commute_res """ select id,min(value1) from max_min_filter_push_down1 group by id having 40>min(value1) order by 1,2; """ From 311b48b1a8d3cce156c276e93d36ebd048ae48a3 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 21 Aug 2024 11:26:27 +0800 Subject: [PATCH 7/9] [Feat](nereids) add max/min filter push down rewrite rule --- .../rules/rewrite/MaxMinFilterPushDown.java | 4 +- .../rewrite/MaxMinFilterPushDownTest.java | 41 +++++++++++++--- .../max_min_filter_push_down.out | 49 +++++++++++-------- .../max_min_filter_push_down.groovy | 10 +++- 4 files changed, 74 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java index c137a5ffa3a97b..a54c3785b35a72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDown.java @@ -63,7 +63,7 @@ public class MaxMinFilterPushDown extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter(logicalAggregate()) + return logicalFilter(logicalAggregate().whenNot(agg -> agg.getGroupByExpressions().isEmpty())) .then(this::pushDownMaxMinFilter) .toRule(RuleType.MAX_MIN_FILTER_PUSH_DOWN); } @@ -110,7 +110,7 @@ private Plan pushDownMaxMinFilter(LogicalFilter> filter) newPredicate = new LessThanEqual(func.child(0), originConjunct.child(1)); } } - Preconditions.checkState(newPredicate != null); + Preconditions.checkState(newPredicate != null, "newPredicate is null"); LogicalFilter newPushDownFilter = new LogicalFilter<>(ImmutableSet.of(newPredicate), aggChild); LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newPushDownFilter)); return PlanUtils.filterOrSelf(newUpperConjuncts, newAgg); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java index 0c8cced405d317..6025acf0d32bc0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java @@ -51,37 +51,64 @@ public void testMinRewrite() { } @Test - public void testMaxNotRewrite0() { + public void testNotRewriteBecauseFuncIsMoreThanOne1() { + String sql = "select id, min(score), max(name) from max_t group by id having min(score)<10 and max(name)>'abc'"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + @Test + public void testNotRewriteBecauseFuncIsMoreThanOne2() { + String sql = "select id, min(score), min(name) from max_t group by id having min(score)<10 and min(name)<'abc'"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } + + @Test + public void testMaxNotRewriteBecauseLessThan() { String sql = "select id, max(score) from max_t group by id having max(score)<10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } @Test - public void testMinNotRewrite1() { + public void testMinNotRewriteBecauseGreaterThan() { String sql = "select id, min(score) from max_t group by id having min(score)>10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } @Test - public void testMinNotRewrite2() { - String sql = "select id, min(score), max(score) from max_t group by id having min(score)>10"; + public void testMinNotRewriteBecauseHasMaxFunc() { + String sql = "select id, min(score), max(score) from max_t group by id having min(score)<10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } @Test - public void testMinNotRewrite3() { - String sql = "select id, min(score), count(score) from max_t group by id having min(score)>10"; + public void testMinNotRewriteBecauseHasCountFunc() { + String sql = "select id, min(score), count(score) from max_t group by id having min(score)<10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } @Test - public void testMinNotRewrite4() { + public void testNotRewriteBecauseConjunctLeftNotSlot() { String sql = "select id, max(score) from max_t group by id having abs(max(score))>10"; PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } + + @Test + public void testRewriteAggFuncHasExpr() { + String sql = "select id, max(score+1) from max_t group by id having max(score+1)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getConjuncts().size() == 1)); + } + + @Test + public void testNotRewriteScalarAgg() { + String sql = "select max(score+1) from max_t having max(score+1)>10"; + PlanChecker.from(connectContext).analyze(sql).rewrite() + .nonMatch(logicalFilter(logicalOlapScan())); + } } diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out index 4ad1baf3387200..0b650210f0a0e2 100644 --- a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -1,4 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !scalar_agg_empty_table -- +PhysicalResultSink +--filter((min(value1) < 20)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalEmptyRelation + -- !min -- PhysicalResultSink --hashAgg[GLOBAL] @@ -78,38 +85,40 @@ PhysicalResultSink -- !min_scalar_agg -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 < 40)) ---------PhysicalOlapScan[max_min_filter_push_down1] +--filter((min(value1) < 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] -- !max_scalar_agg -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 > 40)) ---------PhysicalOlapScan[max_min_filter_push_down1] +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] -- !max_scalar_agg -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 > 40)) ---------PhysicalOlapScan[max_min_filter_push_down1] +--filter((max(value1) > 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] -- !min_equal_scalar_agg -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 <= 20)) ---------PhysicalOlapScan[max_min_filter_push_down1] +--filter((min(value1) <= 20)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] -- !max_equal_scalar_agg -- PhysicalResultSink ---hashAgg[GLOBAL] -----hashAgg[LOCAL] -------filter((max_min_filter_push_down1.value1 >= 40)) ---------PhysicalOlapScan[max_min_filter_push_down1] +--filter((max(value1) >= 40)) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalStorageLayerAggregate[max_min_filter_push_down1] + +-- !scalar_agg_empty_table_res -- -- !min_res -- 1 10 diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy index b1acd02d0e5c6d..635fd180aa9b91 100644 --- a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -32,7 +32,13 @@ suite("max_min_filter_push_down") { INSERT INTO max_min_filter_push_down1 (id, value1, value2) VALUES (1, 10, 'A'),(1, 11, 'A'),(2, 20, 'B'),(2, 73, 'B'),(2, 19, 'B'),(3, 30, 'C'),(3, 61, 'C'),(4, 40, 'D'),(4, 43, 'D'),(4, 45, 'D'); """ + sql "drop table if exists max_min_filter_push_down_empty" + sql "create table max_min_filter_push_down_empty like max_min_filter_push_down1" + qt_scalar_agg_empty_table """ + explain shape plan + select min(value1) from max_min_filter_push_down_empty having min(value1) <40 and min(value1) <20; + """ qt_min """ explain shape plan select id,min(value1) from max_min_filter_push_down1 group by id having min(value1) <40 and min(value1) <20; @@ -105,7 +111,9 @@ suite("max_min_filter_push_down") { select max(value1) from max_min_filter_push_down1 having max(value1) >=40; """ - + qt_scalar_agg_empty_table_res """ + select min(value1) from max_min_filter_push_down_empty having min(value1) <40 and min(value1) <20; + """ qt_min_res """ select id,min(value1) from max_min_filter_push_down1 group by id having min(value1) <40 and min(value1) <20 order by 1,2; """ From 4a62353ea38b7568642752e9402613c1cbdf4cb3 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 21 Aug 2024 11:28:55 +0800 Subject: [PATCH 8/9] fix ut --- .../doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java index 6025acf0d32bc0..bc7d32fb3fb24a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MaxMinFilterPushDownTest.java @@ -56,6 +56,7 @@ public void testNotRewriteBecauseFuncIsMoreThanOne1() { PlanChecker.from(connectContext).analyze(sql).rewrite() .nonMatch(logicalFilter(logicalOlapScan())); } + @Test public void testNotRewriteBecauseFuncIsMoreThanOne2() { String sql = "select id, min(score), min(name) from max_t group by id having min(score)<10 and min(name)<'abc'"; From 879f12fe67643b77540f716f50d21347f3fba234 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 29 Aug 2024 16:45:21 +0800 Subject: [PATCH 9/9] adjust rule position --- .../apache/doris/nereids/jobs/executor/Rewriter.java | 4 +--- .../java/org/apache/doris/nereids/rules/RuleSet.java | 2 ++ .../max_min_filter_push_down.out | 11 +++++++++++ .../max_min_filter_push_down.groovy | 9 ++++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index e62b1cd0d59869..715463b496b3bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -87,7 +87,6 @@ import org.apache.doris.nereids.rules.rewrite.LimitAggToTopNAgg; import org.apache.doris.nereids.rules.rewrite.LimitSortToTopN; import org.apache.doris.nereids.rules.rewrite.LogicalResultSinkToShortCircuitPointQuery; -import org.apache.doris.nereids.rules.rewrite.MaxMinFilterPushDown; import org.apache.doris.nereids.rules.rewrite.MergeAggregate; import org.apache.doris.nereids.rules.rewrite.MergeFilters; import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion; @@ -182,8 +181,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown( // ExtractSingleTableExpressionFromDisjunction conflict to InPredicateToEqualToRule // in the ExpressionNormalization, so must invoke in another job, otherwise dead loop. - new ExtractSingleTableExpressionFromDisjunction(), - new MaxMinFilterPushDown() + new ExtractSingleTableExpressionFromDisjunction() ) ), // subquery unnesting relay on ExpressionNormalization to extract common factor expression diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index be4d8b390c9f1f..e63d4d77bca26e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -88,6 +88,7 @@ import org.apache.doris.nereids.rules.rewrite.ConvertOuterJoinToAntiJoin; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; import org.apache.doris.nereids.rules.rewrite.EliminateOuterJoin; +import org.apache.doris.nereids.rules.rewrite.MaxMinFilterPushDown; import org.apache.doris.nereids.rules.rewrite.MergeFilters; import org.apache.doris.nereids.rules.rewrite.MergeGenerates; import org.apache.doris.nereids.rules.rewrite.MergeLimits; @@ -132,6 +133,7 @@ public class RuleSet { .build(); public static final List PUSH_DOWN_FILTERS = ImmutableList.of( + new MaxMinFilterPushDown(), new CreatePartitionTopNFromWindow(), new PushDownFilterThroughProject(), new PushDownFilterThroughSort(), diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out index 0b650210f0a0e2..2e0ac41d5ebd31 100644 --- a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -118,6 +118,13 @@ PhysicalResultSink ------hashAgg[LOCAL] --------PhysicalStorageLayerAggregate[max_min_filter_push_down1] +-- !depend_prune_column -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------filter((max_min_filter_push_down1.value1 < 10)) +--------PhysicalOlapScan[max_min_filter_push_down1] + -- !scalar_agg_empty_table_res -- -- !min_res -- @@ -187,6 +194,10 @@ PhysicalResultSink -- !max_equal_scalar_agg_res -- 73 +-- !depend_prune_column_res -- +10 +19 + -- !smallint -- PhysicalResultSink --hashAgg[GLOBAL] diff --git a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy index 635fd180aa9b91..47610f2e125701 100644 --- a/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.groovy @@ -111,6 +111,11 @@ suite("max_min_filter_push_down") { select max(value1) from max_min_filter_push_down1 having max(value1) >=40; """ + qt_depend_prune_column """ + explain shape plan + select c1 from (select min(value1) c1,max(value2) from max_min_filter_push_down1 group by id having min(value1)<10) t + """ + qt_scalar_agg_empty_table_res """ select min(value1) from max_min_filter_push_down_empty having min(value1) <40 and min(value1) <20; """ @@ -167,7 +172,9 @@ suite("max_min_filter_push_down") { qt_max_equal_scalar_agg_res """ select max(value1) from max_min_filter_push_down1 having max(value1) >=40; """ - + qt_depend_prune_column_res """ + select c1 from (select min(value1) c1,max(value2) from max_min_filter_push_down1 group by id having min(value1)<20) t order by c1 + """ sql "drop table if exists max_min_filter_push_down2" sql """create table max_min_filter_push_down2(d_int int, d_char100 char(100), d_smallint smallint, d_tinyint tinyint, d_char10 char(10),d_datetimev2 datetimev2, d_datev2 datev2)