From 9796c99324dbc406a69daff6545979fdf25c3f8d Mon Sep 17 00:00:00 2001 From: seawinde Date: Sun, 29 Sep 2024 17:54:55 +0800 Subject: [PATCH 1/2] [fix](mtmv) Fix enable_materialized_view_nest_rewrite session variable is useless in some scene --- .../apache/doris/nereids/CascadesContext.java | 9 + .../doris/nereids/memo/StructInfoMap.java | 16 +- .../mv/AbstractMaterializedViewRule.java | 5 +- .../exploration/mv/MaterializedViewUtils.java | 2 +- .../nested_mtmv_rewrite_switch.groovy | 169 ++++++++++++++++++ 5 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 68812245134e4a..25767134d4d64c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -122,6 +122,7 @@ public class CascadesContext implements ScheduleContext { private final Optional parent; private final Set materializationContexts; + private final Set> materializationRewrittenSuccessSet = new HashSet<>(); private boolean isLeadingJoin = false; private boolean isLeadingDisableJoinReorder = false; @@ -366,6 +367,14 @@ public void addMaterializationContext(MaterializationContext materializationCont this.materializationContexts.add(materializationContext); } + public Set> getMaterializationRewrittenSuccessSet() { + return materializationRewrittenSuccessSet; + } + + public void addMaterializationRewrittenSuccess(List materializationQualifier) { + this.materializationRewrittenSuccessSet.add(materializationQualifier); + } + /** * getAndCacheSessionVariable */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index 0c11b5fbb224ba..4aa4f146b874da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.memo; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; @@ -126,6 +127,9 @@ public void refresh(Group group, CascadesContext cascadesContext) { List> childrenTableMap = new LinkedList<>(); if (groupExpression.children().isEmpty()) { BitSet leaf = constructLeaf(groupExpression, cascadesContext); + if (leaf.isEmpty()) { + break; + } groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>())); continue; } @@ -163,9 +167,19 @@ public void refresh(Group group, CascadesContext cascadesContext) { private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) { Plan plan = groupExpression.getPlan(); BitSet tableMap = new BitSet(); + boolean enableMaterializedViewNestRewrite = cascadesContext.getConnectContext().getSessionVariable() + .isEnableMaterializedViewNestRewrite(); if (plan instanceof LogicalCatalogRelation) { + TableIf table = ((LogicalCatalogRelation) plan).getTable(); + // If disable materialized view nest rewrite, and mv already rewritten successfully once, doesn't construct + // table id map for nest mv rewrite + if (!enableMaterializedViewNestRewrite + && cascadesContext.getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) { + return tableMap; + + } tableMap.set(cascadesContext.getStatementContext() - .getTableId(((LogicalCatalogRelation) plan).getTable()).asInt()); + .getTableId(table).asInt()); } // one row relation / CTE consumer return tableMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 7d84b8ab36b59c..411582344334ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -363,7 +363,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca logicalProperties, queryPlan.getLogicalProperties())); continue; } - recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); + recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext); trySetStatistics(materializationContext, cascadesContext); rewriteResults.add(rewrittenPlan); // if rewrite successfully, try to regenerate mv scan because it maybe used again @@ -852,8 +852,9 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon return checkQueryPattern(structInfo, cascadesContext); } - protected void recordIfRewritten(Plan plan, MaterializationContext context) { + protected void recordIfRewritten(Plan plan, MaterializationContext context, CascadesContext cascadesContext) { context.setSuccess(true); + cascadesContext.addMaterializationRewrittenSuccess(context.generateMaterializationIdentifier()); if (plan.getGroupExpression().isPresent()) { context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 342c88ff677ff0..7bad7f6db280b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -212,8 +212,8 @@ public static List extractStructInfo(Plan plan, Plan originalPlan, C structInfosBuilder.add(structInfo); } } - return structInfosBuilder.build(); } + return structInfosBuilder.build(); } // if plan doesn't belong to any group, construct it directly return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext)); diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy new file mode 100644 index 00000000000000..f9a84fa5250179 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nested_mtmv_rewrite_switch") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_materialized_view_rewrite=true" + + sql """ + drop table if exists orders_2 + """ + sql """ + CREATE TABLE `orders_2` ( + `o_orderkey` BIGINT, + `o_custkey` int, + `o_orderstatus` VARCHAR(1), + `o_totalprice` DECIMAL(15, 2), + `o_orderpriority` VARCHAR(15), + `o_clerk` VARCHAR(15), + `o_shippriority` int, + `o_comment` VARCHAR(79), + `o_orderdate` DATE + ) ENGINE=olap + + PROPERTIES ( + "replication_num" = "1" + + ); + """ + + sql """ + drop table if exists lineitem_2 + """ + sql """ + CREATE TABLE `lineitem_2` ( + `l_orderkey` BIGINT, + `l_linenumber` INT, + `l_partkey` INT, + `l_suppkey` INT, + `l_quantity` DECIMAL(15, 2), + `l_extendedprice` DECIMAL(15, 2), + `l_discount` DECIMAL(15, 2), + `l_tax` DECIMAL(15, 2), + `l_returnflag` VARCHAR(1), + `l_linestatus` VARCHAR(1), + `l_commitdate` DATE, + `l_receiptdate` DATE, + `l_shipinstruct` VARCHAR(25), + `l_shipmode` VARCHAR(10), + `l_comment` VARCHAR(44), + `l_shipdate` DATE + ) ENGINE=olap + + PROPERTIES ( + "replication_num" = "1" + + ); + """ + + sql """ + insert into orders_2 values + (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), + (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'), + (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'), + (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'), + (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), + (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'), + (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); + """ + + sql""" + insert into lineitem_2 values + (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), + (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); + """ + + sql """analyze table orders_2 with sync;""" + sql """analyze table lineitem_2 with sync;""" + + + def compare_res = { def stmt -> + sql "SET enable_materialized_view_rewrite=false" + def origin_res = sql stmt + logger.info("origin_res: " + origin_res) + sql "SET enable_materialized_view_rewrite=true" + def mv_origin_res = sql stmt + logger.info("mv_origin_res: " + mv_origin_res) + assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size())) + for (int row = 0; row < mv_origin_res.size(); row++) { + assertTrue(mv_origin_res[row].size() == origin_res[row].size()) + for (int col = 0; col < mv_origin_res[row].size(); col++) { + assertTrue(mv_origin_res[row][col] == origin_res[row][col]) + } + } + } + + + // create base first level mv + create_async_mv(db, "join_mv1", """ + SELECT l_orderkey, l_linenumber, l_partkey, o_orderkey, o_custkey + FROM lineitem_2 INNER JOIN orders_2 + ON l_orderkey = o_orderkey; + """) + + // create second level mv based on first level mv + create_async_mv(db, "agg_mv2", """ + SELECT + l_orderkey, + l_linenumber, + o_orderkey, + sum(l_partkey) AS total_revenue, + max(o_custkey) AS max_discount + FROM join_mv1 + GROUP BY l_orderkey, l_linenumber, o_orderkey; + """) + + // create third level mv based on second level mv + create_async_mv(db, "join_agg_mv3", """ + SELECT + l_orderkey, + sum(total_revenue) AS total_revenue, + max(max_discount) AS max_discount + FROM agg_mv2 + GROUP BY l_orderkey; + """) + + def query = """ + SELECT l_orderkey, sum(l_partkey) AS total_revenue, max(o_custkey) AS max_discount FROM lineitem_2 INNER JOIN orders_2 ON l_orderkey = o_orderkey GROUP BY l_orderkey + """ + + sql """set enable_materialized_view_nest_rewrite = false;""" + // Just first level mv rewrite successfully, second and third level mv should rewriten fail + mv_rewrite_fail(query, "agg_mv2") + mv_rewrite_fail(query, "join_agg_mv3") + mv_rewrite_success(query, "join_mv1") + compare_res(query + " order by 1,2,3") + + + sql """set enable_materialized_view_nest_rewrite = true;""" + // All mv rewrite successfully but only thirst level mv can be chosen by cbo + mv_rewrite_success_without_check_chosen(query, "join_mv1") + mv_rewrite_success_without_check_chosen(query, "agg_mv2") + mv_rewrite_success(query, "join_agg_mv3") + compare_res(query + " order by 1,2,3") + + +} From 9e968a15d6f976b2fe669fb6733c7ed818a893bb Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 30 Sep 2024 16:52:01 +0800 Subject: [PATCH 2/2] fix regression test --- .../apache/doris/nereids/rules/RuleSet.java | 2 - .../mv/AbstractMaterializedViewRule.java | 8 ++++ .../mv/MaterializedViewOnlyJoinRule.java | 45 ------------------- ...MaterializedViewProjectFilterJoinRule.java | 1 + .../mv/direct_query/direct_query.groovy | 1 + 5 files changed, 10 insertions(+), 47 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 496137118440cf..bcd12ac17d2579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterScanRule; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule; @@ -237,7 +236,6 @@ public class RuleSet { .build(); public static final List MATERIALIZED_VIEW_RULES = planRuleFactories() - .add(MaterializedViewOnlyJoinRule.INSTANCE) .add(MaterializedViewProjectJoinRule.INSTANCE) .add(MaterializedViewFilterJoinRule.INSTANCE) .add(MaterializedViewFilterProjectJoinRule.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 411582344334ac..dd17d754244a15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -37,6 +37,7 @@ import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -354,6 +355,13 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca rewrittenPlanOutput, queryPlan.getOutput())); continue; } + // Merge project + rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext, + childContext -> { + Rewriter.getCteChildrenRewriter(childContext, + ImmutableList.of(Rewriter.bottomUp(new MergeProjects()))).execute(); + return childContext.getRewritePlan(); + }, rewrittenPlan, queryPlan); if (!isOutputValid(queryPlan, rewrittenPlan)) { LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties(); materializationContext.recordFailReason(queryStructInfo, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java deleted file mode 100644 index 2735ca87fe9c39..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; - -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * This is responsible for join pattern such as only join - */ -public class MaterializedViewOnlyJoinRule extends AbstractMaterializedViewJoinRule { - - public static final MaterializedViewOnlyJoinRule INSTANCE = new MaterializedViewOnlyJoinRule(); - - @Override - public List buildRules() { - return ImmutableList.of(logicalJoin(any().when(LogicalPlan.class::isInstance), - any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx -> { - LogicalJoin root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_JOIN)); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java index d82f838ea6b66d..05f54ac3401d55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java @@ -31,6 +31,7 @@ /** * This is responsible for join pattern such as project on filter on join + * Needed because variant data type would have filter on join directly, such as query query3_5 in variant_mv.groovy */ public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule { diff --git a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy index 404154300d2724..7b45349512b174 100644 --- a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy @@ -20,6 +20,7 @@ suite("direct_query_mv") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" sql "set runtime_filter_mode=OFF" + sql """set enable_materialized_view_nest_rewrite = true; """ sql """ drop table if exists orders