From 9f5c97e7bf5f37ee15e23ae8d5f111893f135ac9 Mon Sep 17 00:00:00 2001 From: seawinde Date: Fri, 22 Dec 2023 16:59:22 +0800 Subject: [PATCH 1/4] [fix](nereids) Fix query rewrite fail when query cache build quickly --- .../mv/AbstractMaterializedViewRule.java | 21 ++----------------- .../mv/MaterializationContext.java | 8 +++++++ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 6dcf1885563691..54cc61b708dd9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -26,7 +26,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.BaseTableInfo; -import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; @@ -104,13 +103,8 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { logger.debug(currentClassName + " this group is already rewritten so skip"); continue; } - MTMV mtmv = materializationContext.getMTMV(); - MTMVCache mtmvCache = getCacheFromMTMV(mtmv); - if (mtmvCache == null) { - logger.warn(currentClassName + " mv cache is null so return"); - return rewriteResults; - } - List viewStructInfos = extractStructInfo(mtmvCache.getLogicalPlan(), cascadesContext); + List viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), + cascadesContext); if (viewStructInfos.size() > 1) { // view struct info should only have one logger.warn(currentClassName + " the num of view struct info is more then one so return"); @@ -289,17 +283,6 @@ protected boolean checkPartitionIsValid( && relatedTalbeValidSet.containsAll(relatedTableSelectedPartitionToCheck); } - private MTMVCache getCacheFromMTMV(MTMV mtmv) { - MTMVCache cache; - try { - cache = mtmv.getOrGenerateCache(); - } catch (AnalysisException analysisException) { - logger.warn(this.getClass().getSimpleName() + " get mtmv cache analysisException", analysisException); - return null; - } - return cache; - } - /** * Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 19b1f5be8bb10a..153688ecc2d3dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -51,6 +51,8 @@ public class MaterializationContext { // generate form mv scan plan private ExpressionMapping mvExprToMvScanExprMapping; private boolean available = true; + // the mv plan from cache at present, record it to make sure query rewrite by mv is right when cache change. + private Plan mvPlan; /** * MaterializationContext, this contains necessary info for query rewriting by mv @@ -81,6 +83,8 @@ public MaterializationContext(MTMV mtmv, mtmvCache.getMvOutputExpressions(), mtmvCache.getLogicalPlan()), mvScanPlan.getExpressions()); + // copy the plan from cache, which the plan in cache may change + this.mvPlan = mtmvCache.getLogicalPlan(); } public Set getMatchedGroups() { @@ -119,6 +123,10 @@ public boolean isAvailable() { return available; } + public Plan getMvPlan() { + return mvPlan; + } + /** * MaterializationContext fromMaterializedView */ From ccd458dfbc6ba05f57e34b86e37683e0a5f98ace Mon Sep 17 00:00:00 2001 From: seawinde Date: Sat, 23 Dec 2023 20:38:46 +0800 Subject: [PATCH 2/4] improve the regression test and fix mv plan rewrite --- .../mv/AbstractMaterializedViewRule.java | 2 +- .../aggregate_with_roll_up.groovy | 33 +++++++++++++++++-- .../aggregate_without_roll_up.groovy | 21 ++++++++---- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 54cc61b708dd9d..90ebe567c4628e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -194,7 +194,7 @@ protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { CascadesContext rewrittenPlanContext = CascadesContext.initContext(cascadesContext.getStatementContext(), rewrittenPlan, cascadesContext.getCurrentJobContext().getRequiredProperties()); - Rewriter.getWholeTreeRewriter(cascadesContext).execute(); + Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); rewrittenPlan = rewrittenPlanContext.getRewritePlan(); logger.debug(currentClassName + "rewrite by materialized view success"); rewriteResults.add(rewrittenPlan); diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index e67086deba0668..b56922d5db6470 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -42,7 +42,11 @@ suite("aggregate_with_roll_up") { O_COMMENT VARCHAR(79) NOT NULL ) DUPLICATE KEY(o_orderkey, o_custkey) - PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" @@ -73,7 +77,10 @@ suite("aggregate_with_roll_up") { l_comment VARCHAR(44) NOT NULL ) DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) - PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(l_shipdate) ( + PARTITION `day_1` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_2` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_3` VALUES LESS THAN ("2023-12-30")) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" @@ -144,6 +151,26 @@ suite("aggregate_with_roll_up") { } } + def check_rewrite_with_mv_partition = { mv_sql, query_sql, mv_name, partition_column -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (${partition_column}) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + explain { + sql("${query_sql}") + contains "(${mv_name})" + } + } + def check_rewrite_with_force_analyze = { mv_sql, query_sql, mv_name -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" @@ -283,7 +310,7 @@ suite("aggregate_with_roll_up") { "l_partkey, " + "l_suppkey" order_qt_query15_0_before "${query15_0}" - check_rewrite(mv15_0, query15_0, "mv15_0") + check_rewrite_with_mv_partition(mv15_0, query15_0, "mv15_0", "l_shipdate") order_qt_query15_0_after "${query15_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv15_0""" diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index b76763752fbd36..192575d50f59f6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -43,11 +43,15 @@ suite("aggregate_without_roll_up") { o_comment VARCHAR(79) NOT NULL ) DUPLICATE KEY(o_orderkey, o_custkey) - PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-8'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-10"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" - ) + ); """ sql """ @@ -74,7 +78,11 @@ suite("aggregate_without_roll_up") { l_comment VARCHAR(44) NOT NULL ) DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) - PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN ('2023-12-30')) + PARTITION BY RANGE(l_shipdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-8'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-10"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" @@ -100,7 +108,8 @@ suite("aggregate_without_roll_up") { ) """ - sql """ insert into lineitem values + sql """ + insert into lineitem values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), @@ -110,8 +119,8 @@ suite("aggregate_without_roll_up") { sql """ insert into orders values - (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), - (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 9.5, '2023-12-07', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-07', 'a', 'b', 1, 'yy'), (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), From a98e4398046eed969d936669e007bcfff403bdc8 Mon Sep 17 00:00:00 2001 From: seawinde Date: Sat, 23 Dec 2023 21:40:37 +0800 Subject: [PATCH 3/4] change init data --- .../aggregate_without_roll_up.groovy | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 192575d50f59f6..9bfbf96b00a472 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -44,8 +44,8 @@ suite("aggregate_without_roll_up") { ) DUPLICATE KEY(o_orderkey, o_custkey) PARTITION BY RANGE(o_orderdate) ( - PARTITION `day_2` VALUES LESS THAN ('2023-12-8'), - PARTITION `day_3` VALUES LESS THAN ("2023-12-10"), + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), PARTITION `day_4` VALUES LESS THAN ("2023-12-30") ) DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 @@ -79,8 +79,8 @@ suite("aggregate_without_roll_up") { ) DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) PARTITION BY RANGE(l_shipdate) ( - PARTITION `day_2` VALUES LESS THAN ('2023-12-8'), - PARTITION `day_3` VALUES LESS THAN ("2023-12-10"), + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), PARTITION `day_4` VALUES LESS THAN ("2023-12-30") ) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 @@ -119,8 +119,6 @@ suite("aggregate_without_roll_up") { sql """ insert into orders values - (1, 1, 'o', 9.5, '2023-12-07', 'a', 'b', 1, 'yy'), - (1, 1, 'o', 10.5, '2023-12-07', 'a', 'b', 1, 'yy'), (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), From 809cf6705122024a8ab04bed41b7d766522c8007 Mon Sep 17 00:00:00 2001 From: seawinde Date: Sun, 24 Dec 2023 01:44:47 +0800 Subject: [PATCH 4/4] roll back init data --- .../mv/agg_without_roll_up/aggregate_without_roll_up.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 9bfbf96b00a472..72e80eadbf93f0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -119,6 +119,8 @@ suite("aggregate_without_roll_up") { sql """ insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),