From bb7ab388c540b2630ab9023e798b8867a33bd720 Mon Sep 17 00:00:00 2001 From: yujun Date: Tue, 25 Feb 2025 16:35:12 +0800 Subject: [PATCH 1/7] fix merge projects with non-foldable expression multiple times --- .../post/MergeProjectPostProcessor.java | 2 +- .../nereids/processor/post/Validator.java | 7 ------ .../rules/exploration/MergeProjectsCBO.java | 1 + .../rewrite/DeferMaterializeTopNResult.java | 2 +- .../nereids/rules/rewrite/MergeProjects.java | 5 ++-- .../nereids/trees/plans/algebra/Project.java | 10 ++++++++ .../apache/doris/nereids/util/PlanUtils.java | 24 +++++++++++++++++++ 7 files changed, 39 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java index d900d207b2f22d..7466889fde64ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java @@ -33,7 +33,7 @@ public class MergeProjectPostProcessor extends PlanPostProcessor { public PhysicalProject visitPhysicalProject(PhysicalProject project, CascadesContext ctx) { project = (PhysicalProject) super.visit(project, ctx); Plan child = project.child(); - if (child instanceof PhysicalProject) { + if (child instanceof PhysicalProject && project.canMergeProjections((PhysicalProject) child)) { List projections = project.mergeProjections((PhysicalProject) child); return (PhysicalProject) project .withProjectionsAndChild(projections, child.child(0)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java index 6fe5f0e1c3ed86..1240c68c1e0cc0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java @@ -44,13 +44,6 @@ public class Validator extends PlanPostProcessor { @Override public Plan visitPhysicalProject(PhysicalProject project, CascadesContext context) { Preconditions.checkArgument(!project.getProjects().isEmpty(), "Project list can't be empty"); - - Plan child = project.child(); - // Forbidden project-project, we must merge project. - if (child instanceof PhysicalProject) { - throw new AnalysisException("Nereids must merge a project-project plan"); - } - return visit(project, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/MergeProjectsCBO.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/MergeProjectsCBO.java index ff55596722e989..4637425b55dee8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/MergeProjectsCBO.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/MergeProjectsCBO.java @@ -28,6 +28,7 @@ public class MergeProjectsCBO extends OneExplorationRuleFactory { @Override public Rule build() { return logicalProject(logicalProject()) + .when(project -> project.canMergeProjections(project.child())) .then(project -> MergeProjects.mergeProjects(project)) .toRule(RuleType.MERGE_PROJECTS); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java index b606489c7730df..34a8dd631395b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java @@ -150,7 +150,7 @@ public List buildRules() { } return true; }) - )).then(r -> { + ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> { LogicalProject upperProject = r.child(); LogicalProject> bottomProject = r.child().child().child(); List projections = upperProject.mergeProjections(bottomProject); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java index 77784253440fab..bb6ca154136a63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.ExpressionUtils; import java.util.List; @@ -43,11 +42,11 @@ public Rule build() { // TODO modify ExtractAndNormalizeWindowExpression to handle nested window functions // here we just don't merge two projects if there is any window function return logicalProject(logicalProject()) - .whenNot(project -> ExpressionUtils.containsWindowExpression(project.getProjects()) - && ExpressionUtils.containsWindowExpression(project.child().getProjects())) + .when(project -> project.canMergeProjections(project.child())) .then(MergeProjects::mergeProjects).toRule(RuleType.MERGE_PROJECTS); } + /** merge projects */ public static Plan mergeProjects(LogicalProject project) { LogicalProject childProject = (LogicalProject) project.child(); List projectExpressions = project.mergeProjections(childProject); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java index 606253148358a0..65fbf3f036e989 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java @@ -75,6 +75,16 @@ default List mergeProjections(Project childProject) { return projects; } + /** check can merge two projects */ + default boolean canMergeProjections(Project childProject) { + if (ExpressionUtils.containsWindowExpression(getProjects()) + && ExpressionUtils.containsWindowExpression(childProject.getProjects())) { + return false; + } + + return PlanUtils.canReplaceWithProjections(childProject.getProjects(), getProjects()); + } + /** * find projects, if not found the slot, then throw AnalysisException */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java index 7438e46ddc7966..4d1ba233b9b013 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -45,6 +45,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -133,6 +134,29 @@ public static List replaceExpressionByProjections(List childProjects, + List targetExpressions) { + Set nonfoldableSlots = ExpressionUtils.generateReplaceMap(childProjects).entrySet().stream() + .filter(entry -> entry.getValue().containsNonfoldable()) + .map(Entry::getKey) + .collect(Collectors.toSet()); + if (nonfoldableSlots.isEmpty()) { + return true; + } + + Set counterSet = Sets.newHashSet(); + return targetExpressions.stream().noneMatch(target -> target.anyMatch( + e -> (e instanceof Slot) && nonfoldableSlots.contains(e) && !counterSet.add((Slot) e))); + } + public static Plan skipProjectFilterLimit(Plan plan) { if (plan instanceof LogicalProject && ((LogicalProject) plan).isAllSlots() || plan instanceof LogicalFilter || plan instanceof LogicalLimit) { From e2f2f044087435f2af7a71b4a2b2a7ac20f111fc Mon Sep 17 00:00:00 2001 From: yujun Date: Tue, 25 Feb 2025 18:46:16 +0800 Subject: [PATCH 2/7] add test --- .../nereids_rules_p0/test_nonfoldable.groovy | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy diff --git a/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy new file mode 100644 index 00000000000000..f70558470772a6 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_nonfoldable') { + sql 'SET enable_nereids_planner=true' + sql 'SET runtime_filter_mode=OFF' + sql 'SET enable_fallback_to_original_planner=false' + sql "SET ignore_shape_nodes='PhysicalDistribute'" + sql 'SET disable_nereids_rules=PRUNE_EMPTY_PARTITION' + + qt_filter_through_project_1 ''' + explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 + ''' + + qt_filter_through_project_2 ''' + explain physical plan select * from (select id + random(1, 10) + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 + ''' + + qt_filter_through_project_3 ''' + explain physical plan select * from (select id + random(1, 10) + 100 as a, id + random(1, 10) + 200 as b, id + random(1, 10) + 300 as c from t1) t where a > 999 and b > 999 + ''' + + qt_filter_through_project_4 ''' + explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a + random(1, 10) > 999 and b + random(1, 10) > 999 + ''' + + qt_filter_through_project_5 ''' + explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 limit 10 + ''' + + qt_filter_through_project_6 ''' + explain physical plan select * from (select id + random(1, 10) + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 limit 10 + ''' + + qt_filter_through_project_7 ''' + explain physical plan select * from (select id + random(1, 10) + 100 as a, id + random(1, 10) + 200 as b, id + random(1, 10) + 300 as c from t1) t where a > 999 and b > 999 limit 10 + ''' + + qt_filter_through_project_8 ''' + explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a + random(1, 10) > 999 and b + random(1, 10) > 999 limit 10 + ''' + + qt_merge_project_1 ''' + explain physical plan select a as b, a as c from (select id + 100 as a from t1) t + ''' + + qt_merge_project_2 ''' + explain physical plan select a as b, a as c from (select id + random(1, 10) as a from t1) t + ''' + + qt_merge_project_3 ''' + explain physical plan select a as b from (select id + random(1, 10) as a from t1) t + ''' + + qt_merge_project_3 ''' + explain physical plan select a + 10 + a as b from (select id + random(1, 10) as a from t1) t + ''' + + qt_merge_project_3 ''' + explain physical plan select a as b, a + 10 as c from (select id + random(1, 10) as a from t1) t + ''' +} From 9314ef17243660543d53577912de4bf6cf3e7f94 Mon Sep 17 00:00:00 2001 From: yujun Date: Tue, 25 Feb 2025 18:58:12 +0800 Subject: [PATCH 3/7] run forceGenOut --- .../nereids_rules_p0/test_nonfoldable.out | 87 +++++++++++++++++++ .../nereids_rules_p0/test_nonfoldable.groovy | 30 +++---- 2 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/test_nonfoldable.out diff --git a/regression-test/data/nereids_rules_p0/test_nonfoldable.out b/regression-test/data/nereids_rules_p0/test_nonfoldable.out new file mode 100644 index 00000000000000..a6dfc43a1ca669 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/test_nonfoldable.out @@ -0,0 +1,87 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !filter_through_project_1 -- +PhysicalResultSink +--PhysicalProject +----filter((cast(id as BIGINT) > 899)) +------PhysicalOlapScan[t1] + +-- !filter_through_project_2 -- +PhysicalResultSink +--filter((t.a > 999)) +----PhysicalProject +------filter((cast(id as BIGINT) > 799)) +--------PhysicalOlapScan[t1] + +-- !filter_through_project_3 -- +PhysicalResultSink +--filter((t.a > 999) and (t.b > 999)) +----PhysicalProject +------PhysicalOlapScan[t1] + +-- !filter_through_project_4 -- +PhysicalResultSink +--PhysicalProject +----filter(((t1.id + random(1, 10)) > 899)) +------PhysicalOlapScan[t1] + +-- !filter_through_project_5 -- +PhysicalResultSink +--PhysicalLimit[GLOBAL] +----PhysicalLimit[LOCAL] +------PhysicalProject +--------filter((cast(id as BIGINT) > 899)) +----------PhysicalOlapScan[t1] + +-- !filter_through_project_6 -- +PhysicalResultSink +--PhysicalLimit[GLOBAL] +----PhysicalLimit[LOCAL] +------filter((t.a > 999)) +--------PhysicalProject +----------filter((cast(id as BIGINT) > 799)) +------------PhysicalOlapScan[t1] + +-- !filter_through_project_7 -- +PhysicalResultSink +--PhysicalLimit[GLOBAL] +----PhysicalLimit[LOCAL] +------filter((t.a > 999) and (t.b > 999)) +--------PhysicalProject +----------PhysicalOlapScan[t1] + +-- !filter_through_project_8 -- +PhysicalResultSink +--PhysicalLimit[GLOBAL] +----PhysicalLimit[LOCAL] +------PhysicalProject +--------filter(((t1.id + random(1, 10)) > 899)) +----------PhysicalOlapScan[t1] + +-- !merge_project_1 -- +PhysicalResultSink +--PhysicalProject +----PhysicalOlapScan[t1] + +-- !merge_project_2 -- +PhysicalResultSink +--PhysicalProject +----PhysicalProject +------PhysicalOlapScan[t1] + +-- !merge_project_3 -- +PhysicalResultSink +--PhysicalProject +----PhysicalOlapScan[t1] + +-- !merge_project_4 -- +PhysicalResultSink +--PhysicalProject +----PhysicalProject +------PhysicalOlapScan[t1] + +-- !merge_project_5 -- +PhysicalResultSink +--PhysicalProject +----PhysicalProject +------PhysicalOlapScan[t1] + diff --git a/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy index f70558470772a6..bbd88cdde3d8cf 100644 --- a/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy +++ b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy @@ -23,54 +23,54 @@ suite('test_nonfoldable') { sql 'SET disable_nereids_rules=PRUNE_EMPTY_PARTITION' qt_filter_through_project_1 ''' - explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 + explain shape plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 ''' qt_filter_through_project_2 ''' - explain physical plan select * from (select id + random(1, 10) + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 + explain shape plan select * from (select id + random(1, 10) + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 ''' qt_filter_through_project_3 ''' - explain physical plan select * from (select id + random(1, 10) + 100 as a, id + random(1, 10) + 200 as b, id + random(1, 10) + 300 as c from t1) t where a > 999 and b > 999 + explain shape plan select * from (select id + random(1, 10) + 100 as a, id + random(1, 10) + 200 as b, id + random(1, 10) + 300 as c from t1) t where a > 999 and b > 999 ''' qt_filter_through_project_4 ''' - explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a + random(1, 10) > 999 and b + random(1, 10) > 999 + explain shape plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a + random(1, 10) > 999 and b + random(1, 10) > 999 ''' qt_filter_through_project_5 ''' - explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 limit 10 + explain shape plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 limit 10 ''' qt_filter_through_project_6 ''' - explain physical plan select * from (select id + random(1, 10) + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 limit 10 + explain shape plan select * from (select id + random(1, 10) + 100 as a, id + 200 as b, id + 300 as c from t1) t where a > 999 and b > 999 limit 10 ''' qt_filter_through_project_7 ''' - explain physical plan select * from (select id + random(1, 10) + 100 as a, id + random(1, 10) + 200 as b, id + random(1, 10) + 300 as c from t1) t where a > 999 and b > 999 limit 10 + explain shape plan select * from (select id + random(1, 10) + 100 as a, id + random(1, 10) + 200 as b, id + random(1, 10) + 300 as c from t1) t where a > 999 and b > 999 limit 10 ''' qt_filter_through_project_8 ''' - explain physical plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a + random(1, 10) > 999 and b + random(1, 10) > 999 limit 10 + explain shape plan select * from (select id + 100 as a, id + 200 as b, id + 300 as c from t1) t where a + random(1, 10) > 999 and b + random(1, 10) > 999 limit 10 ''' qt_merge_project_1 ''' - explain physical plan select a as b, a as c from (select id + 100 as a from t1) t + explain shape plan select a as b, a as c from (select id + 100 as a from t1) t ''' qt_merge_project_2 ''' - explain physical plan select a as b, a as c from (select id + random(1, 10) as a from t1) t + explain shape plan select a as b, a as c from (select id + random(1, 10) as a from t1) t ''' qt_merge_project_3 ''' - explain physical plan select a as b from (select id + random(1, 10) as a from t1) t + explain shape plan select a as b from (select id + random(1, 10) as a from t1) t ''' - qt_merge_project_3 ''' - explain physical plan select a + 10 + a as b from (select id + random(1, 10) as a from t1) t + qt_merge_project_4 ''' + explain shape plan select a + 10 + a as b from (select id + random(1, 10) as a from t1) t ''' - qt_merge_project_3 ''' - explain physical plan select a as b, a + 10 as c from (select id + random(1, 10) as a from t1) t + qt_merge_project_5 ''' + explain shape plan select a as b, a + 10 as c from (select id + random(1, 10) as a from t1) t ''' } From 4a27941f9332195e3c62d7603876ebf539e3f9d8 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 26 Feb 2025 11:05:26 +0800 Subject: [PATCH 4/7] add session variable detail_shape_plan_nodes --- .../trees/plans/physical/PhysicalProject.java | 19 +++++++++++++++++++ .../org/apache/doris/qe/SessionVariable.java | 19 +++++++++++++++++++ .../nereids_rules_p0/test_nonfoldable.groovy | 1 + 3 files changed, 39 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java index 8dfb5cf55ddc6d..7bb1b92ce2edd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.Statistics; import com.google.common.base.Preconditions; @@ -49,6 +50,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Physical project plan. @@ -100,6 +102,23 @@ public String toString() { ); } + @Override + public String shapeInfo() { + ConnectContext context = ConnectContext.get(); + if (context != null + && context.getSessionVariable().getDetailShapePlanNodesSet().contains(getClass().getSimpleName())) { + StringBuilder builder = new StringBuilder(); + builder.append(getClass().getSimpleName()); + // the internal project list's order may be unstable, especial for join tables, + // so sort the projects to make it stable + builder.append(projects.stream().map(Expression::shapeInfo).sorted() + .collect(Collectors.joining(", ", "[", "]"))); + return builder.toString(); + } else { + return super.shapeInfo(); + } + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 48a3aefcc30749..74ea7a74e970b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2202,6 +2202,8 @@ public void setIgnoreRuntimeFilterIds(String ignoreRuntimeFilterIds) { public static final String IGNORE_SHAPE_NODE = "ignore_shape_nodes"; + public static final String DETAIL_SHAPE_NODES = "detail_shape_nodes"; + public static final String ENABLE_SEGMENT_CACHE = "enable_segment_cache"; public Set getIgnoreShapePlanNodes() { @@ -2217,6 +2219,23 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { "the plan node type which is ignored in 'explain shape plan' command"}) public String ignoreShapePlanNodes = ""; + @VariableMgr.VarAttr(name = DETAIL_SHAPE_NODES, needForward = true, setter = "setDetailShapeNodes", + description = {"'explain shape plan' 命令中显示详细信息的PlanNode 类型", + "the plan node type show detail in 'explain shape plan' command"}) + public String detailShapePlanNodes = ""; + + private Set detailShapePlanNodesSet = ImmutableSet.of(); + + public Set getDetailShapePlanNodesSet() { + return detailShapePlanNodesSet; + } + + public void setDetailShapePlanNodesSet(String detailShapePlanNodes) { + this.detailShapePlanNodesSet = Arrays.stream(ignoreShapePlanNodes.split(",[\\s]*")) + .collect(ImmutableSet.toImmutableSet()); + this.detailShapePlanNodes = detailShapePlanNodes; + } + @VariableMgr.VarAttr(name = ENABLE_DECIMAL256, needForward = true, description = { "控制是否在计算过程中使用Decimal256类型", "Set to true to enable Decimal256 type" }) public boolean enableDecimal256 = false; diff --git a/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy index bbd88cdde3d8cf..ab2d19fd2b810e 100644 --- a/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy +++ b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy @@ -20,6 +20,7 @@ suite('test_nonfoldable') { sql 'SET runtime_filter_mode=OFF' sql 'SET enable_fallback_to_original_planner=false' sql "SET ignore_shape_nodes='PhysicalDistribute'" + set "Set detail_shape_nodes='PhysicalProject'" sql 'SET disable_nereids_rules=PRUNE_EMPTY_PARTITION' qt_filter_through_project_1 ''' From 1bff671e24b9f67ca583883c1d625497bcf17a31 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 26 Feb 2025 11:46:52 +0800 Subject: [PATCH 5/7] fix seter --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 4 ++-- .../suites/nereids_rules_p0/test_nonfoldable.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 74ea7a74e970b0..4764c76a496f70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2219,7 +2219,7 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { "the plan node type which is ignored in 'explain shape plan' command"}) public String ignoreShapePlanNodes = ""; - @VariableMgr.VarAttr(name = DETAIL_SHAPE_NODES, needForward = true, setter = "setDetailShapeNodes", + @VariableMgr.VarAttr(name = DETAIL_SHAPE_NODES, needForward = true, setter = "setDetailShapePlanNodes", description = {"'explain shape plan' 命令中显示详细信息的PlanNode 类型", "the plan node type show detail in 'explain shape plan' command"}) public String detailShapePlanNodes = ""; @@ -2230,7 +2230,7 @@ public Set getDetailShapePlanNodesSet() { return detailShapePlanNodesSet; } - public void setDetailShapePlanNodesSet(String detailShapePlanNodes) { + public void setDetailShapePlanNodes(String detailShapePlanNodes) { this.detailShapePlanNodesSet = Arrays.stream(ignoreShapePlanNodes.split(",[\\s]*")) .collect(ImmutableSet.toImmutableSet()); this.detailShapePlanNodes = detailShapePlanNodes; diff --git a/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy index ab2d19fd2b810e..f48d93f986ef26 100644 --- a/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy +++ b/regression-test/suites/nereids_rules_p0/test_nonfoldable.groovy @@ -20,7 +20,7 @@ suite('test_nonfoldable') { sql 'SET runtime_filter_mode=OFF' sql 'SET enable_fallback_to_original_planner=false' sql "SET ignore_shape_nodes='PhysicalDistribute'" - set "Set detail_shape_nodes='PhysicalProject'" + sql "SET detail_shape_nodes='PhysicalProject'" sql 'SET disable_nereids_rules=PRUNE_EMPTY_PARTITION' qt_filter_through_project_1 ''' From 9d4f3c7909d4fd8a23a51f5bdefa60e93f132f6a Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 26 Feb 2025 11:56:37 +0800 Subject: [PATCH 6/7] fix --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 4764c76a496f70..e69f3b616e8bfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2231,7 +2231,7 @@ public Set getDetailShapePlanNodesSet() { } public void setDetailShapePlanNodes(String detailShapePlanNodes) { - this.detailShapePlanNodesSet = Arrays.stream(ignoreShapePlanNodes.split(",[\\s]*")) + this.detailShapePlanNodesSet = Arrays.stream(detailShapePlanNodes.split(",[\\s]*")) .collect(ImmutableSet.toImmutableSet()); this.detailShapePlanNodes = detailShapePlanNodes; } From 01c112b95c32e6f3ff1bc9fd9013b65766ccb9f8 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 26 Feb 2025 12:06:42 +0800 Subject: [PATCH 7/7] run forceGenOut --- .../nereids_rules_p0/test_nonfoldable.out | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/test_nonfoldable.out b/regression-test/data/nereids_rules_p0/test_nonfoldable.out index a6dfc43a1ca669..3c96406efb6052 100644 --- a/regression-test/data/nereids_rules_p0/test_nonfoldable.out +++ b/regression-test/data/nereids_rules_p0/test_nonfoldable.out @@ -1,26 +1,26 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !filter_through_project_1 -- PhysicalResultSink ---PhysicalProject +--PhysicalProject[(id + 100) AS `a`, (id + 200) AS `b`, (id + 300) AS `c`] ----filter((cast(id as BIGINT) > 899)) ------PhysicalOlapScan[t1] -- !filter_through_project_2 -- PhysicalResultSink --filter((t.a > 999)) -----PhysicalProject +----PhysicalProject[((id + random(1, 10)) + 100) AS `a`, (id + 200) AS `b`, (id + 300) AS `c`] ------filter((cast(id as BIGINT) > 799)) --------PhysicalOlapScan[t1] -- !filter_through_project_3 -- PhysicalResultSink --filter((t.a > 999) and (t.b > 999)) -----PhysicalProject +----PhysicalProject[((id + random(1, 10)) + 100) AS `a`, ((id + random(1, 10)) + 200) AS `b`, ((id + random(1, 10)) + 300) AS `c`] ------PhysicalOlapScan[t1] -- !filter_through_project_4 -- PhysicalResultSink ---PhysicalProject +--PhysicalProject[(id + 100) AS `a`, (id + 200) AS `b`, (id + 300) AS `c`] ----filter(((t1.id + random(1, 10)) > 899)) ------PhysicalOlapScan[t1] @@ -28,7 +28,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] -------PhysicalProject +------PhysicalProject[(id + 100) AS `a`, (id + 200) AS `b`, (id + 300) AS `c`] --------filter((cast(id as BIGINT) > 899)) ----------PhysicalOlapScan[t1] @@ -37,7 +37,7 @@ PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------filter((t.a > 999)) ---------PhysicalProject +--------PhysicalProject[((id + random(1, 10)) + 100) AS `a`, (id + 200) AS `b`, (id + 300) AS `c`] ----------filter((cast(id as BIGINT) > 799)) ------------PhysicalOlapScan[t1] @@ -46,42 +46,42 @@ PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------filter((t.a > 999) and (t.b > 999)) ---------PhysicalProject +--------PhysicalProject[((id + random(1, 10)) + 100) AS `a`, ((id + random(1, 10)) + 200) AS `b`, ((id + random(1, 10)) + 300) AS `c`] ----------PhysicalOlapScan[t1] -- !filter_through_project_8 -- PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] -------PhysicalProject +------PhysicalProject[(id + 100) AS `a`, (id + 200) AS `b`, (id + 300) AS `c`] --------filter(((t1.id + random(1, 10)) > 899)) ----------PhysicalOlapScan[t1] -- !merge_project_1 -- PhysicalResultSink ---PhysicalProject +--PhysicalProject[(id + 100) AS `b`, (id + 100) AS `c`] ----PhysicalOlapScan[t1] -- !merge_project_2 -- PhysicalResultSink ---PhysicalProject -----PhysicalProject +--PhysicalProject[a AS `b`, a AS `c`] +----PhysicalProject[(id + random(1, 10)) AS `a`] ------PhysicalOlapScan[t1] -- !merge_project_3 -- PhysicalResultSink ---PhysicalProject +--PhysicalProject[(id + random(1, 10)) AS `b`] ----PhysicalOlapScan[t1] -- !merge_project_4 -- PhysicalResultSink ---PhysicalProject -----PhysicalProject +--PhysicalProject[((a + a) + 10) AS `b`] +----PhysicalProject[(id + random(1, 10)) AS `a`] ------PhysicalOlapScan[t1] -- !merge_project_5 -- PhysicalResultSink ---PhysicalProject -----PhysicalProject +--PhysicalProject[(a + 10) AS `c`, a AS `b`] +----PhysicalProject[(id + random(1, 10)) AS `a`] ------PhysicalOlapScan[t1]