From b0b25d1392beef2b9376ec032c96808980cad364 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 21 May 2024 16:33:19 +0800 Subject: [PATCH] 1 --- .../java/org/apache/doris/catalog/MTMV.java | 39 ++++++++++++ .../java/org/apache/doris/mtmv/MTMVTest.java | 61 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index d95c2388a288d4..1b4a8e7063e73f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.catalog.OlapTableFactory.MTMVParams; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.job.common.TaskStatus; @@ -316,6 +317,44 @@ public Map generateMvPartitionDescs() { return result; } + /** + * Calculate the partition and associated partition mapping relationship of the MTMV + * It is the result of real-time comparison calculation, so there may be some costs, + * so it should be called with caution. + * The reason for not directly calling `calculatePartitionMappings` and + * generating a reverse index is to directly generate two maps here, + * without the need to traverse them again + * + * @return mvPartitionName ==> relationPartitionNames and relationPartitionName ==> mvPartitionName + * @throws AnalysisException + */ + public Pair>, Map> calculateDoublyPartitionMappings() + throws AnalysisException { + if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { + return Pair.of(Maps.newHashMap(), Maps.newHashMap()); + } + long start = System.currentTimeMillis(); + Map> mvToBase = Maps.newHashMap(); + Map baseToMv = Maps.newHashMap(); + Map> relatedPartitionDescs = MTMVPartitionUtil + .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties); + Map mvPartitionItems = getAndCopyPartitionItems(); + for (Entry entry : mvPartitionItems.entrySet()) { + Set basePartitionNames = relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), + Sets.newHashSet()); + String mvPartitionName = entry.getKey(); + mvToBase.put(mvPartitionName, basePartitionNames); + for (String basePartitionName : basePartitionNames) { + baseToMv.put(basePartitionName, mvPartitionName); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("calculateDoublyPartitionMappings use [{}] mills, mvName is [{}]", + System.currentTimeMillis() - start, name); + } + return Pair.of(mvToBase, baseToMv); + } + /** * Calculate the partition and associated partition mapping relationship of the MTMV * It is the result of real-time comparison calculation, so there may be some costs, diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java index 4a3bf12f8899ea..f004124d50d96e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java @@ -17,18 +17,33 @@ package org.apache.doris.mtmv; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; import org.apache.doris.job.common.IntervalUnit; import org.apache.doris.job.extensions.mtmv.MTMVTask; import org.apache.doris.mtmv.MTMVRefreshEnum.BuildMode; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshTrigger; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.junit.Assert; import org.junit.Test; import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; public class MTMVTest { @Test @@ -83,4 +98,50 @@ private MTMVTask buildMTMVTask(MTMV mtmv) { task.setTaskId(1L); return task; } + + @Test + public void testCalculateDoublyPartitionMappings() throws AnalysisException { + Map> mvToBase = Maps.newHashMap(); + Map baseToMv = Maps.newHashMap(); + Map> relatedPartitionDescs = mockRelatedPartitionDescs(); + Map mvPartitionItems = mockMvPartitionItems(); + for (Entry entry : mvPartitionItems.entrySet()) { + Set basePartitionNames = relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), + Sets.newHashSet()); + String mvPartitionName = entry.getKey(); + mvToBase.put(mvPartitionName, basePartitionNames); + for (String basePartitionName : basePartitionNames) { + baseToMv.put(basePartitionName, mvPartitionName); + } + } + Assert.assertEquals(mvToBase.get("mvp1"), Sets.newHashSet("baseP1_1", "baseP1_2")); + Assert.assertEquals(baseToMv.get("baseP1_1"), "mvp1"); + Assert.assertEquals(baseToMv.get("baseP1_2"), "mvp1"); + } + + private Map> mockRelatedPartitionDescs() throws AnalysisException { + Map> res = Maps.newHashMap(); + Column k1 = new Column("k1", ScalarType.createType(PrimitiveType.TINYINT), true, null, "", "key1"); + PartitionKey rangeP1Lower = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("1")), + Lists.newArrayList(k1)); + PartitionKey rangeP1Upper = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")), + Lists.newArrayList(k1)); + Range rangeP1 = Range.closedOpen(rangeP1Lower, rangeP1Upper); + PartitionItem item1 = new RangePartitionItem(rangeP1); + res.put(item1.toPartitionKeyDesc(), Sets.newHashSet("baseP1_1", "baseP1_2")); + return res; + } + + private Map mockMvPartitionItems() throws AnalysisException { + Map res = Maps.newHashMap(); + Column k1 = new Column("k1", ScalarType.createType(PrimitiveType.TINYINT), true, null, "", "key1"); + PartitionKey rangeP1Lower = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("1")), + Lists.newArrayList(k1)); + PartitionKey rangeP1Upper = PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue("10")), + Lists.newArrayList(k1)); + Range rangeP1 = Range.closedOpen(rangeP1Lower, rangeP1Upper); + PartitionItem item1 = new RangePartitionItem(rangeP1); + res.put("mvp1", item1); + return res; + } }