From e91cdb3c15c9de740cb994c80ee168a3b84d3d09 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 13 Mar 2025 12:28:40 +0800 Subject: [PATCH] [fix](mtmv) Fix nest mtmv rewrite fail when bottom mtmv cache is invalid (#48222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What problem does this PR solve? Fix nest mtmv rewrite fail when bottom mtmv cache is invalid such as bottom mv is mv_1 and mv_2 as following： **mv_1** select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, sum(o_totalprice) as sum_total, max(o_totalprice) as max_total, min(o_totalprice) as min_total, count(*) as count_all, bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 from lineitem_1 inner join orders_1 on lineitem_1.l_orderkey = orders_1.o_orderkey where lineitem_1.l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey **mv_2** select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, t.agg1 as agg1, t.sum_total as agg3, t.max_total as agg4, t.min_total as agg5, t.count_all as agg6, cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 from ${mv_1} as t inner join partsupp_1 on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey where partsupp_1.ps_suppkey > 1 group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 **mv_3** select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 from ${mv_2} as t1 left join ${mv_2} as t2 on t1.l_orderkey = t2.l_orderkey where t1.l_orderkey > 1 group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 ``` query as following would fail if mtmvCache invalid in mv_1 and mv_2, the pr fix this ```sql select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 from ( select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, t.agg1 as agg1, t.sum_total as agg3, t.max_total as agg4, t.min_total as agg5, t.count_all as agg6, cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 from ( select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, sum(o_totalprice) as sum_total, max(o_totalprice) as max_total, min(o_totalprice) as min_total, count(*) as count_all, bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 from lineitem_1 inner join orders_1 on lineitem_1.l_orderkey = orders_1.o_orderkey where lineitem_1.l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey ) as t inner join partsupp_1 on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey where partsupp_1.ps_suppkey > 1 group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 ) as t1 left join ( select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, t.agg1 as agg1, t.sum_total as agg3, t.max_total as agg4, t.min_total as agg5, t.count_all as agg6, cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 from ( select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, sum(o_totalprice) as sum_total, max(o_totalprice) as max_total, min(o_totalprice) as min_total, count(*) as count_all, bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 from lineitem_1 inner join orders_1 on lineitem_1.l_orderkey = orders_1.o_orderkey where lineitem_1.l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey ) as t inner join partsupp_1 on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey where partsupp_1.ps_suppkey > 1 group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 ) as t2 on t1.l_orderkey = t2.l_orderkey where t1.l_orderkey > 1 group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 --- .../java/org/apache/doris/catalog/MTMV.java | 4 -- .../trees/plans/logical/LogicalOlapScan.java | 54 +++++++++++++++---- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index c540ee5c5417dd..d96a4997a80e12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -332,10 +332,6 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws Ana } } - public MTMVCache getCache() { - return cache; - } - public Map getMvProperties() { readMvLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 2216e58c4fa3b8..a34fa9b356e26e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DataTrait; @@ -46,6 +47,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONObject; import java.util.ArrayList; @@ -62,6 +65,8 @@ */ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan { + private static final Logger LOG = LogManager.getLogger(LogicalOlapScan.class); + /////////////////////////////////////////////////////////////////////////// // Members for materialized index. /////////////////////////////////////////////////////////////////////////// @@ -527,9 +532,15 @@ AGGREGATE KEY (siteid,citycode,username) Set outputSet = Utils.fastToImmutableSet(getOutputSet()); if (getTable() instanceof MTMV) { MTMV mtmv = (MTMV) getTable(); - MTMVCache cache = mtmv.getCache(); + MTMVCache cache; + try { + cache = mtmv.getOrGenerateCache(ConnectContext.get()); + } catch (AnalysisException e) { + LOG.warn(String.format("LogicalOlapScan computeUnique fail, mv name is %s", mtmv.getName()), e); + return; + } // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index - if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { + if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { return; } Plan originalPlan = cache.getOriginalPlan(); @@ -554,9 +565,15 @@ AGGREGATE KEY (siteid,citycode,username) public void computeUniform(DataTrait.Builder builder) { if (getTable() instanceof MTMV) { MTMV mtmv = (MTMV) getTable(); - MTMVCache cache = mtmv.getCache(); + MTMVCache cache; + try { + cache = mtmv.getOrGenerateCache(ConnectContext.get()); + } catch (AnalysisException e) { + LOG.warn(String.format("LogicalOlapScan computeUniform fail, mv name is %s", mtmv.getName()), e); + return; + } // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index - if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { + if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { return; } Plan originalPlan = cache.getOriginalPlan(); @@ -569,9 +586,15 @@ public void computeUniform(DataTrait.Builder builder) { public void computeEqualSet(DataTrait.Builder builder) { if (getTable() instanceof MTMV) { MTMV mtmv = (MTMV) getTable(); - MTMVCache cache = mtmv.getCache(); + MTMVCache cache; + try { + cache = mtmv.getOrGenerateCache(ConnectContext.get()); + } catch (AnalysisException e) { + LOG.warn(String.format("LogicalOlapScan computeEqualSet fail, mv name is %s", mtmv.getName()), e); + return; + } // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index - if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { + if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { return; } Plan originalPlan = cache.getOriginalPlan(); @@ -584,9 +607,15 @@ public void computeEqualSet(DataTrait.Builder builder) { public void computeFd(DataTrait.Builder builder) { if (getTable() instanceof MTMV) { MTMV mtmv = (MTMV) getTable(); - MTMVCache cache = mtmv.getCache(); + MTMVCache cache; + try { + cache = mtmv.getOrGenerateCache(ConnectContext.get()); + } catch (AnalysisException e) { + LOG.warn(String.format("LogicalOlapScan computeFd fail, mv name is %s", mtmv.getName()), e); + return; + } // Maybe query specified index, should not calc, such as select count(*) from t1 index col_index - if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { + if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) { return; } Plan originalPlan = cache.getOriginalPlan(); @@ -599,7 +628,14 @@ Map constructReplaceMap(MTMV mtmv) { Map replaceMap = new HashMap<>(); // Need remove invisible column, and then mapping them List originOutputs = new ArrayList<>(); - for (Slot originSlot : mtmv.getCache().getOriginalPlan().getOutput()) { + MTMVCache cache; + try { + cache = mtmv.getOrGenerateCache(ConnectContext.get()); + } catch (AnalysisException e) { + LOG.warn(String.format("LogicalOlapScan constructReplaceMap fail, mv name is %s", mtmv.getName()), e); + return replaceMap; + } + for (Slot originSlot : cache.getOriginalPlan().getOutput()) { if (!(originSlot instanceof SlotReference) || (((SlotReference) originSlot).isVisible())) { originOutputs.add(originSlot); }