From 22e3b58df385302c3c548fda6926f815b4ad478c Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 26 Nov 2025 16:20:52 +0800 Subject: [PATCH 1/5] [fix] (mtmv) Fix topN rewrite by materialized view fail and cause rewrite result wrong --- .../mv/AbstractMaterializedViewRule.java | 6 +- .../mv/AbstractMaterializedViewScanRule.java | 4 +- .../mv/MaterializedViewLimitScanRule.java | 5 ++ .../mv/MaterializedViewTopNScanRule.java | 5 ++ .../exploration/mv/MaterializedViewUtils.java | 20 +++++ .../MaterializedViewWindowAggregateRule.java | 9 +++ .../mv/MaterializedViewWindowJoinRule.java | 9 +++ .../mv/MaterializedViewWindowScanRule.java | 9 +++ .../mv/MaterializedViewUtilsTest.java | 75 +++++++++++++++++++ .../nereids_rules_p0/mv/topN/topN_rewrite.out | 12 +++ .../mv/topN/topN_rewrite.groovy | 32 ++++++++ 11 files changed, 182 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 6668b363897e07..69232d6e261b93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -1066,9 +1066,9 @@ protected Plan tryRewriteTopN(LogicalTopN queryTopNode, LogicalTopN // check the order keys of TopN between query and view is consistent List queryOrderKeys = queryTopNode.getOrderKeys(); List viewOrderKeys = viewTopNode.getOrderKeys(); - if (queryOrderKeys.size() != viewOrderKeys.size()) { + if (queryOrderKeys.size() > viewOrderKeys.size()) { materializationContext.recordFailReason(queryStructInfo, - "query topN order keys size is not consistent with view topN order keys size", + "query topN order keys size is bigger than view topN order keys size", () -> String.format("query topN order keys = %s,\n view topN order keys = %s,\n", queryOrderKeys, viewOrderKeys)); return null; @@ -1096,7 +1096,7 @@ protected Plan tryRewriteTopN(LogicalTopN queryTopNode, LogicalTopN viewShuttledOrderKeys.add(new OrderKey(viewOrderByExpressionsQueryBasedSet.get(j), viewOrderKey.isAsc(), viewOrderKey.isNullFirst())); } - if (!queryShuttledOrderKeys.equals(viewShuttledOrderKeys)) { + if (!MaterializedViewUtils.isPrefixSameFromStart(queryShuttledOrderKeys, viewShuttledOrderKeys)) { materializationContext.recordFailReason(queryStructInfo, "view topN order key doesn't match query order key", () -> String.format("queryShuttledOrderKeys = %s,\n viewShuttledOrderKeys = %s,\n", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java index a91e2314d3248c..24c63361f69d68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java @@ -90,6 +90,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) - && !checkContext.isContainsTopAggregate(); + && !checkContext.isContainsTopAggregate() + && !checkContext.isContainsTopLimit() && !checkContext.isContainsTopTopN() + && !checkContext.isContainsTopWindow(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java index 8059a39bf687e2..92d75836802af5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java @@ -74,6 +74,11 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca && checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1; } + @Override + protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { + return checkQueryPattern(structInfo, cascadesContext); + } + @Override public List buildRules() { return ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java index 27fb752361aabb..9fc73c5eac7bbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java @@ -72,6 +72,11 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca && checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1; } + @Override + protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { + return checkQueryPattern(structInfo, cascadesContext); + } + @Override public List buildRules() { return ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 347b800c096d3a..319703cfa07a96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.StructInfoMap; +import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindRelation; import org.apache.doris.nereids.rules.exploration.mv.PartitionIncrementMaintainer.PartitionIncrementCheckContext; @@ -651,4 +652,23 @@ public Boolean visit(Plan plan, Void context) { return false; } } + + /** + * Check the prefix of two order key list is same from start + */ + public static boolean isPrefixSameFromStart(List queryShuttledOrderKeys, + List viewShuttledOrderKeys) { + if (queryShuttledOrderKeys == null || viewShuttledOrderKeys == null) { + return false; + } + if (queryShuttledOrderKeys.size() > viewShuttledOrderKeys.size()) { + return false; + } + for (int i = 0; i < queryShuttledOrderKeys.size(); i++) { + if (!java.util.Objects.equals(queryShuttledOrderKeys.get(i), viewShuttledOrderKeys.get(i))) { + return false; + } + } + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java index f1b6905e2c3c7b..ae909b74f44938 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java @@ -45,6 +45,15 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf tempRewritedPlan, materializationContext, cascadesContext); } + @Override + protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { + PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); + return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() + && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() + && !checkContext.isContainsTopLimit(); + } + /** * Check window pattern is valid or not */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java index f1f05d25cc3343..a3a62e52b1a685 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java @@ -57,6 +57,15 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca && checkContext.getTopWindowNum() <= 1; } + @Override + protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { + PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); + return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() + && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() + && !checkContext.isContainsTopLimit(); + } + @Override public List buildRules() { return ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java index 01ffb0ad351053..5c4a73a284915d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java @@ -58,6 +58,15 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca && checkContext.getTopWindowNum() <= 1; } + @Override + protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { + PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); + return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) + && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() + && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() + && !checkContext.isContainsTopLimit(); + } + @Override public List buildRules() { return ImmutableList.of( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index ae225aec5e9a3c..1c6bbcfc25ef09 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -20,14 +20,19 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.exploration.mv.RelatedTableInfo.RelatedTableColumnInfo; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; + /** * Test for materialized view util */ @@ -952,6 +957,76 @@ public void getRelatedTableInfoWhenMultiPartitionExprs() { }); } + // java + @Test + public void isPrefixSameFromStartNullAndEmptyTests() { + // both null + Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(null, null)); + // query null + List view = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false) + ); + Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(null, view)); + // view null + List query = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false) + ); + Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(query, null)); + List emptyQuery = ImmutableList.of(); + List emptyView = ImmutableList.of(); + Assertions.assertTrue(MaterializedViewUtils.isPrefixSameFromStart(emptyQuery, emptyView)); + } + + @Test + public void isPrefixSameFromStart_queryLongerThanView() { + List query = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false), + new OrderKey(new VarcharLiteral("b"), true, false) + ); + List view = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false) + ); + Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(query, view)); + } + + @Test + public void isPrefixSameFromStart_emptyQueryIsPrefix() { + List emptyQuery = ImmutableList.of(); + List view = ImmutableList.of( + new OrderKey(new VarcharLiteral("x"), true, false), + new OrderKey(new VarcharLiteral("y"), true, false) + ); + Assertions.assertTrue(MaterializedViewUtils.isPrefixSameFromStart(emptyQuery, view)); + } + + @Test + public void isPrefixSameFromStart_prefixEqualReturnsTrue() { + List query = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false), + new OrderKey(new VarcharLiteral("b"), true, false) + ); + List view = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false), + new OrderKey(new VarcharLiteral("b"), true, false), + new OrderKey(new VarcharLiteral("c"), true, false) + ); + Assertions.assertTrue(MaterializedViewUtils.isPrefixSameFromStart(query, view)); + } + + @Test + public void isPrefixSameFromStart_prefixNotEqualReturnsFalse() { + List query = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false), + new OrderKey(new VarcharLiteral("x"), true, false) + ); + List view = ImmutableList.of( + new OrderKey(new VarcharLiteral("a"), true, false), + new OrderKey(new VarcharLiteral("b"), true, false), + new OrderKey(new VarcharLiteral("c"), true, false) + ); + Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(query, view)); + } + private void checkRelatedTableInfo(RelatedTableInfo relatedTableInfo, String expectTableName, String expectColumnName, diff --git a/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out b/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out index 90dc8e849db4e6..ef3e5153eecbe2 100644 --- a/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out +++ b/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out @@ -9,6 +9,18 @@ 2023-12-12 1 1 5 1 2023-12-13 1 1 6 1 +-- !query1_3_before -- +2023-12-10 2 3 +2023-12-11 3 4 +2023-12-12 2 5 +2023-12-13 2 6 + +-- !query1_3_after -- +2023-12-10 2 3 +2023-12-11 3 4 +2023-12-12 2 5 +2023-12-13 2 6 + -- !query1_1_before -- 2023-12-12 1 1 5 1 2023-12-13 1 1 6 1 diff --git a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy index 54dcab2dcd0ae3..9e1e5b3081bd94 100644 --- a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy @@ -182,6 +182,38 @@ suite("topN_rewrite") { order_qt_query1_0_after "${query1_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + def mv1_3 = + """ + select + o_orderdate, + ps_partkey, + l_orderkey + from + orders left + join lineitem on l_orderkey = o_orderkey + left join partsupp on ps_partkey = l_partkey and l_suppkey = ps_suppkey + order by o_orderdate, l_orderkey, ps_partkey + limit 8 offset 1; + """ + def query1_3 = + """ + select + o_orderdate, + ps_partkey, + l_orderkey + from + orders left + join lineitem on l_orderkey = o_orderkey + left join partsupp on ps_partkey = l_partkey and l_suppkey = ps_suppkey + order by o_orderdate, l_orderkey + limit 4 offset 2; + """ + order_qt_query1_3_before "${query1_3}" + // query top order by is subset of mv order by columns and prefix is same, should success + async_mv_rewrite_success(db, mv1_3, query1_3, "mv1_3") + order_qt_query1_3_after "${query1_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" + def mv1_1 = """ select From 7b486ff9de888855172bc71b8471cb606642c36b Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 26 Nov 2025 16:53:55 +0800 Subject: [PATCH 2/5] add test --- .../nereids_rules_p0/mv/topN/topN_rewrite.out | 10 ++++++ .../mv/topN/topN_rewrite.groovy | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out b/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out index ef3e5153eecbe2..d6655fbed6dce5 100644 --- a/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out +++ b/regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out @@ -21,6 +21,16 @@ 2023-12-12 2 5 2023-12-13 2 6 +-- !query1_4_before -- +2023-12-11 3 4 +2023-12-12 2 5 +2023-12-13 2 6 + +-- !query1_4_after -- +2023-12-11 3 4 +2023-12-12 2 5 +2023-12-13 2 6 + -- !query1_1_before -- 2023-12-12 1 1 5 1 2023-12-13 1 1 6 1 diff --git a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy index 9e1e5b3081bd94..fe1e7115448843 100644 --- a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy @@ -214,6 +214,39 @@ suite("topN_rewrite") { order_qt_query1_3_after "${query1_3}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" + def mv1_4 = + """ + select + o_orderdate, + ps_partkey, + l_orderkey + from + orders left + join lineitem on l_orderkey = o_orderkey + left join partsupp on ps_partkey = l_partkey and l_suppkey = ps_suppkey + order by o_orderdate, l_orderkey, ps_partkey + limit 8 offset 1; + """ + def query1_4 = + """ + select + o_orderdate, + ps_partkey, + l_orderkey + from + orders left + join lineitem on l_orderkey = o_orderkey + left join partsupp on ps_partkey = l_partkey and l_suppkey = ps_suppkey + where o_orderdate > '2023-12-08' + order by o_orderdate, l_orderkey + limit 4 offset 2; + """ + order_qt_query1_4_before "${query1_4}" + // query top order by is subset of mv order by columns and prefix is same, should success + async_mv_rewrite_fail(db, mv1_4, query1_4, "mv1_4") + order_qt_query1_4_after "${query1_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4""" + def mv1_1 = """ select From c36992033aa0b87d3c299463a41f999423f46e7c Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 26 Nov 2025 16:54:21 +0800 Subject: [PATCH 3/5] fix test --- .../suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy index fe1e7115448843..72485b902e228f 100644 --- a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy @@ -242,7 +242,7 @@ suite("topN_rewrite") { limit 4 offset 2; """ order_qt_query1_4_before "${query1_4}" - // query top order by is subset of mv order by columns and prefix is same, should success + // should fail because of the filter async_mv_rewrite_fail(db, mv1_4, query1_4, "mv1_4") order_qt_query1_4_after "${query1_4}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4""" From bd412f13e017e613a8953016dc9728811d653ffb Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 27 Nov 2025 11:45:39 +0800 Subject: [PATCH 4/5] fix code --- ...MaterializedViewAggregateOnNoneAggregateRule.java | 8 ++++++-- .../mv/MaterializedViewLimitAggregateRule.java | 1 + .../mv/MaterializedViewLimitJoinRule.java | 1 + .../mv/MaterializedViewLimitScanRule.java | 1 + .../mv/MaterializedViewTopNAggregateRule.java | 1 + .../exploration/mv/MaterializedViewTopNJoinRule.java | 1 + .../exploration/mv/MaterializedViewTopNScanRule.java | 1 + .../mv/MaterializedViewWindowAggregateRule.java | 12 ++---------- .../mv/MaterializedViewWindowJoinRule.java | 12 ++---------- .../mv/MaterializedViewWindowScanRule.java | 12 ++---------- 10 files changed, 18 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java index 813136b067d7fa..12a822b73885ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java @@ -82,13 +82,17 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon // any check result of join or scan is true, then return true PlanCheckContext joinCheckContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); boolean joinCheckResult = structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, joinCheckContext) - && !joinCheckContext.isContainsTopAggregate(); + && !joinCheckContext.isContainsTopAggregate() + && !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN() + && !joinCheckContext.isContainsTopWindow(); if (joinCheckResult) { return true; } PlanCheckContext scanCheckContext = PlanCheckContext.of(ImmutableSet.of()); return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, scanCheckContext) - && !scanCheckContext.isContainsTopAggregate(); + && !scanCheckContext.isContainsTopAggregate() + && !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN() + && !joinCheckContext.isContainsTopWindow(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java index deb35055e2461d..575e8f406d5768 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java @@ -70,6 +70,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) && checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1 && !checkContext.isContainsTopTopN() + && !checkContext.isContainsTopWindow() && checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java index 1f61931d7d54ff..2bfe8fe59efb42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java @@ -70,6 +70,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopTopN() + && !checkContext.isContainsTopWindow() && checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java index 92d75836802af5..b230d7fbd33d88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java @@ -71,6 +71,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext) && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopTopN() + && !checkContext.isContainsTopWindow() && checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java index a0e80ebb4960b8..7b96f37ecdc708 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java @@ -67,6 +67,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) && checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1 && !checkContext.isContainsTopLimit() + && !checkContext.isContainsTopWindow() && checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java index ebf0e5318677e2..7c7d6b89fcb2f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java @@ -68,6 +68,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca return accept && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopLimit() + && !checkContext.isContainsTopWindow() && checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java index 9fc73c5eac7bbd..43f5dcf10fb935 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java @@ -69,6 +69,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca return accept && !checkContext.isContainsTopAggregate() && !checkContext.isContainsTopLimit() + && !checkContext.isContainsTopWindow() && checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java index ae909b74f44938..7a0dd68692db69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java @@ -45,15 +45,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf tempRewritedPlan, materializationContext, cascadesContext); } - @Override - protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) - && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() - && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() - && !checkContext.isContainsTopLimit(); - } - /** * Check window pattern is valid or not */ @@ -63,7 +54,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) && checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() && checkContext.getTopAggregateNum() <= 1 && checkContext.getTopWindowNum() <= 1 - && !checkContext.isWindowUnderAggregate(); + && !checkContext.isWindowUnderAggregate() + && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java index a3a62e52b1a685..01175d25f78972 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java @@ -54,16 +54,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() - && checkContext.getTopWindowNum() <= 1; - } - - @Override - protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) - && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() - && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() - && !checkContext.isContainsTopLimit(); + && checkContext.getTopWindowNum() <= 1 + && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java index 5c4a73a284915d..6df3316887a7a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java @@ -55,16 +55,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext) && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() - && checkContext.getTopWindowNum() <= 1; - } - - @Override - protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) { - PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); - return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) - && !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow() - && checkContext.getTopWindowNum() <= 1 && !checkContext.isContainsTopTopN() - && !checkContext.isContainsTopLimit(); + && checkContext.getTopWindowNum() <= 1 + && !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit(); } @Override From 24a868b778f832cfbc65fef5f3508f2815037b1f Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 1 Dec 2025 18:01:59 +0800 Subject: [PATCH 5/5] fix regression test --- .../nereids_rules_p0/mv/limit/limit_rewrite.groovy | 9 ++++++--- .../suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy index 5971de6301767e..bc31601d78b898 100644 --- a/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy @@ -580,7 +580,8 @@ suite("limit_rewrite") { where o_orderdate > '2023-12-08' limit 2 offset 5; """ - async_mv_rewrite_success(db, mv5_1, query5_1, "mv5_1") + // mv data can not cover query limit offset, should fail + async_mv_rewrite_fail(db, mv5_1, query5_1, "mv5_1") sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_1""" @@ -606,7 +607,8 @@ suite("limit_rewrite") { where o_orderdate > '2023-12-09' limit 4 offset 2; """ - async_mv_rewrite_success(db, mv5_2, query5_2, "mv5_2") + // mv data can not cover query limit offset, should fail + async_mv_rewrite_fail(db, mv5_2, query5_2, "mv5_2") sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_2""" @@ -657,7 +659,8 @@ suite("limit_rewrite") { orders limit 2 offset 5; """ - async_mv_rewrite_success(db, mv6_1, query6_1, "mv6_1") + // mv data can not cover query limit offset, should fail + async_mv_rewrite_fail(db, mv6_1, query6_1, "mv6_1") sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1""" } diff --git a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy index 72485b902e228f..5ddd3cff73ee9c 100644 --- a/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/topN/topN_rewrite.groovy @@ -658,7 +658,8 @@ suite("topN_rewrite") { limit 2 offset 5; """ order_qt_query5_1_before "${query5_1}" - async_mv_rewrite_success(db, mv5_1, query5_1, "mv5_1") + // mv data can not cover query limit offset, should fail + async_mv_rewrite_fail(db, mv5_1, query5_1, "mv5_1") order_qt_query5_1_after "${query5_1}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_1""" @@ -689,7 +690,8 @@ suite("topN_rewrite") { limit 4 offset 2; """ order_qt_query5_2_before "${query5_2}" - async_mv_rewrite_success(db, mv5_2, query5_2, "mv5_2") + // mv data can not cover query limit offset, should fail + async_mv_rewrite_fail(db, mv5_2, query5_2, "mv5_2") order_qt_query5_2_after "${query5_2}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_2""" @@ -748,7 +750,8 @@ suite("topN_rewrite") { limit 2 offset 5; """ order_qt_query6_1_before "${query6_1}" - async_mv_rewrite_success(db, mv6_1, query6_1, "mv6_1") + // mv data can not cover query limit offset, should fail + async_mv_rewrite_fail(db, mv6_1, query6_1, "mv6_1") order_qt_query6_1_after "${query6_1}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1"""