From 035f03ae1de3912e6d50455f3376406e904036d9 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 6 Nov 2024 19:06:08 +0800 Subject: [PATCH 1/3] 1 --- .../org/apache/doris/catalog/OlapTable.java | 5 +++- .../doris/common/proc/PartitionsProcDir.java | 30 +++++++++++-------- .../apache/doris/mtmv/MTMVPartitionUtil.java | 4 +-- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index b0d27ac7b5c7d9..c876d6245188bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -113,6 +113,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -3246,7 +3247,9 @@ public PartitionType getPartitionType() { @Override public Map getAndCopyPartitionItems() { - readLock(); + if (!tryReadLock(1, TimeUnit.MINUTES)) { + throw new RuntimeException("get table read lock timeout, database=" + getDBName() + ",table=" + getName()); + } try { Map res = Maps.newHashMap(); for (Entry entry : getPartitionInfo().getIdToItem(false).entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index f1aed5b6278960..ce602bc27a4d0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -241,6 +241,16 @@ private List, TRow>> getPartitionInfosInrernal() throws An // get info List, TRow>> partitionInfos = new ArrayList, TRow>>(); + Map> partitionsUnSyncTables = null; + String mtmvPartitionSyncErrorMsg = null; + if (olapTable instanceof MTMV) { + try { + partitionsUnSyncTables = MTMVPartitionUtil + .getPartitionsUnSyncTables((MTMV) olapTable); + } catch (AnalysisException e) { + mtmvPartitionSyncErrorMsg = e.getMessage(); + } + } olapTable.readLock(); try { List partitionIds; @@ -258,16 +268,6 @@ private List, TRow>> getPartitionInfosInrernal() throws An } Joiner joiner = Joiner.on(", "); - Map> partitionsUnSyncTables = null; - String mtmvPartitionSyncErrorMsg = null; - if (olapTable instanceof MTMV) { - try { - partitionsUnSyncTables = MTMVPartitionUtil - .getPartitionsUnSyncTables((MTMV) olapTable, partitionIds); - } catch (AnalysisException e) { - mtmvPartitionSyncErrorMsg = e.getMessage(); - } - } for (Long partitionId : partitionIds) { Partition partition = olapTable.getPartition(partitionId); @@ -363,11 +363,15 @@ private List, TRow>> getPartitionInfosInrernal() throws An if (StringUtils.isEmpty(mtmvPartitionSyncErrorMsg)) { List partitionUnSyncTables = partitionsUnSyncTables.getOrDefault(partitionId, Lists.newArrayList()); - boolean isSync = CollectionUtils.isEmpty(partitionUnSyncTables); + boolean isSync = partitionsUnSyncTables.containsKey(partitionId) && CollectionUtils.isEmpty(partitionUnSyncTables); partitionInfo.add(isSync); trow.addToColumnValue(new TCell().setBoolVal(isSync)); - partitionInfo.add(partitionUnSyncTables.toString()); - trow.addToColumnValue(new TCell().setStringVal(partitionUnSyncTables.toString())); + // The calculation logic of partitionsUnSyncTables is not protected in the current lock, + // so the obtained partition list may not be consistent with here + String unSyncTables = partitionsUnSyncTables.containsKey(partitionId) + ? partitionUnSyncTables.toString() : "not sure, please try again"; + partitionInfo.add(unSyncTables); + trow.addToColumnValue(new TCell().setStringVal(unSyncTables)); } else { partitionInfo.add(false); trow.addToColumnValue(new TCell().setBoolVal(false)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 2a0863a6e61101..1cfb5e021a5309 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -231,12 +231,12 @@ public static boolean isMTMVSync(MTMVRefreshContext context, Set * getPartitionsUnSyncTables * * @param mtmv - * @param partitionIds * @return partitionName ==> UnSyncTableNames * @throws AnalysisException */ - public static Map> getPartitionsUnSyncTables(MTMV mtmv, List partitionIds) + public static Map> getPartitionsUnSyncTables(MTMV mtmv) throws AnalysisException { + List partitionIds = mtmv.getPartitionIds(); Map> res = Maps.newHashMap(); MTMVRefreshContext context = MTMVRefreshContext.buildContext(mtmv); for (Long partitionId : partitionIds) { From 6c37abfa119060c7a541816c9fb022ec7b80d74b Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 6 Nov 2024 19:53:03 +0800 Subject: [PATCH 2/3] change exception to AnalysisException --- fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java | 2 +- .../src/main/java/org/apache/doris/catalog/OlapTable.java | 4 ++-- .../main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 93d9a8d8dfb1ee..b0d25ad2b252b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -358,7 +358,7 @@ public MTMVRefreshSnapshot getRefreshSnapshot() { * * @return mvPartitionName ==> mvPartitionKeyDesc */ - public Map generateMvPartitionDescs() { + public Map generateMvPartitionDescs() throws AnalysisException { Map mtmvItems = getAndCopyPartitionItems(); Map result = Maps.newHashMap(); for (Entry entry : mtmvItems.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index c876d6245188bc..f01c27863efc26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3246,9 +3246,9 @@ public PartitionType getPartitionType() { } @Override - public Map getAndCopyPartitionItems() { + public Map getAndCopyPartitionItems() throws AnalysisException { if (!tryReadLock(1, TimeUnit.MINUTES)) { - throw new RuntimeException("get table read lock timeout, database=" + getDBName() + ",table=" + getName()); + throw new AnalysisException("get table read lock timeout, database=" + getDBName() + ",table=" + getName()); } try { Map res = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java index 516eb904e58463..0b7562747745f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java @@ -38,7 +38,7 @@ public interface MTMVRelatedTableIf extends TableIf { * * @return partitionId->PartitionItem */ - Map getAndCopyPartitionItems(); + Map getAndCopyPartitionItems() throws AnalysisException; /** * getPartitionType LIST/RANGE/UNPARTITIONED From c1a411df084606834beea9df3bb736eff1a467fa Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 7 Nov 2024 10:23:48 +0800 Subject: [PATCH 3/3] 1 --- .../java/org/apache/doris/common/proc/PartitionsProcDir.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index ce602bc27a4d0d..c76a4185fca6c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -363,7 +363,8 @@ private List, TRow>> getPartitionInfosInrernal() throws An if (StringUtils.isEmpty(mtmvPartitionSyncErrorMsg)) { List partitionUnSyncTables = partitionsUnSyncTables.getOrDefault(partitionId, Lists.newArrayList()); - boolean isSync = partitionsUnSyncTables.containsKey(partitionId) && CollectionUtils.isEmpty(partitionUnSyncTables); + boolean isSync = partitionsUnSyncTables.containsKey(partitionId) && CollectionUtils.isEmpty( + partitionUnSyncTables); partitionInfo.add(isSync); trow.addToColumnValue(new TCell().setBoolVal(isSync)); // The calculation logic of partitionsUnSyncTables is not protected in the current lock,