From 3b401b073e35507eb202ab5c4057051da91289c8 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 30 Jul 2024 16:03:11 +0800 Subject: [PATCH 1/3] 1 --- .../doris/datasource/hive/HMSExternalTable.java | 9 ++++----- .../doris/datasource/hive/HiveMetaStoreCache.java | 11 +++++++++++ .../org/apache/doris/datasource/CatalogMgrTest.java | 1 + 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index d34d8cf769bfbc..e398dee9a20254 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -759,11 +759,10 @@ public Map getAndCopyPartitionItems() { @Override public String getPartitionName(long partitionId) throws AnalysisException { - Map partitionNameToIdMap = getHivePartitionValues().getPartitionNameToIdMap(); - for (Entry entry : partitionNameToIdMap.entrySet()) { - if (entry.getValue().equals(partitionId)) { - return entry.getKey(); - } + Map partitionIdToNameMap = getHivePartitionValues().getPartitionIdToNameMap(); + String name = partitionIdToNameMap.get(partitionId); + if (name != null) { + return name; } throw new AnalysisException("can not find partition, partitionId: " + partitionId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 7f23385d847886..7007f4fc5094db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -1087,6 +1087,7 @@ public static class HiveFileStatus { public static class HivePartitionValues { private long nextPartitionId; private Map partitionNameToIdMap; + private Map partitionIdToNameMap; private Map> idToUniqueIdsMap; private Map idToPartitionItem; private Map> partitionValuesMap; @@ -1115,6 +1116,7 @@ public HivePartitionValues(Map idToPartitionItem, this.singleColumnRangeMap = singleColumnRangeMap; this.nextPartitionId = nextPartitionId; this.partitionNameToIdMap = partitionNameToIdMap; + this.partitionIdToNameMap = transferToIdName(partitionNameToIdMap); this.idToUniqueIdsMap = idToUniqueIdsMap; this.singleUidToColumnRangeMap = singleUidToColumnRangeMap; this.partitionValuesMap = partitionValuesMap; @@ -1124,6 +1126,7 @@ public HivePartitionValues copy() { HivePartitionValues copy = new HivePartitionValues(); copy.setNextPartitionId(nextPartitionId); copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : Maps.newHashMap(partitionNameToIdMap)); + copy.setPartitionIdToNameMap(partitionIdToNameMap == null ? null : Maps.newHashMap(partitionIdToNameMap)); copy.setIdToUniqueIdsMap(idToUniqueIdsMap == null ? null : Maps.newHashMap(idToUniqueIdsMap)); copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap)); @@ -1138,5 +1141,13 @@ public HivePartitionValues copy() { } return copy; } + + private Map transferToIdName(Map partitionNameToIdMap) { + if (partitionNameToIdMap == null) { + return null; + } + return partitionNameToIdMap.entrySet().stream() + .collect(Collectors.toMap(entry -> entry.getValue(), entry -> entry.getKey())); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 364e1f23b16de1..8a6bb03646cab4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -477,6 +477,7 @@ public void testAddSingleColumnPartitionsCache() { partitionValueCacheKey.getTypes()); HivePartitionValues partitionValues = metaStoreCache.getPartitionValues(partitionValueCacheKey); Assert.assertEquals(partitionValues.getPartitionNameToIdMap().size(), 4); + Assert.assertEquals(partitionValues.getPartitionIdToNameMap().size(), 4); } @Test From f7f7d0d0e54b29d6bd70eea2fefd044cb0a003fb Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 30 Jul 2024 17:58:14 +0800 Subject: [PATCH 2/3] 1 --- .../org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 7007f4fc5094db..379b7fa8ddde2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -636,6 +636,7 @@ public void addPartitionsCache(String dbName, String tblName, List parti HivePartitionValues copy = partitionValues.copy(); Map idToPartitionItemBefore = copy.getIdToPartitionItem(); Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); + Map partitionIdToNameMapBefore = copy.getPartitionIdToNameMap(); Map> idToUniqueIdsMap = copy.getIdToUniqueIdsMap(); Map idToPartitionItem = new HashMap<>(); long idx = copy.getNextPartitionId(); @@ -649,6 +650,7 @@ public void addPartitionsCache(String dbName, String tblName, List parti idToPartitionItemBefore.put(partitionId, listPartitionItem); idToPartitionItem.put(partitionId, listPartitionItem); partitionNameToIdMapBefore.put(partitionName, partitionId); + partitionIdToNameMapBefore.put(partitionId, partitionName); } Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); From 26271ce27b64657f154a67953f07ebdf2894dc27 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 5 Aug 2024 17:50:44 +0800 Subject: [PATCH 3/3] use bi map --- .../org/apache/doris/catalog/OlapTable.java | 10 -------- .../datasource/hive/HMSExternalTable.java | 19 +++------------ .../datasource/hive/HiveMetaStoreCache.java | 23 +++++-------------- .../apache/doris/mtmv/MTMVPartitionUtil.java | 9 -------- .../apache/doris/mtmv/MTMVRelatedTableIf.java | 9 -------- .../doris/datasource/CatalogMgrTest.java | 6 +++-- .../doris/mtmv/MTMVPartitionUtilTest.java | 8 ------- 7 files changed, 13 insertions(+), 71 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 916dda773e3d7d..47a4d6fa52c7db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2919,16 +2919,6 @@ public MTMVSnapshotIf getTableSnapshot() { return new MTMVVersionSnapshot(visibleVersion); } - @Override - public String getPartitionName(long partitionId) throws AnalysisException { - readLock(); - try { - return getPartitionOrAnalysisException(partitionId).getName(); - } finally { - readUnlock(); - } - } - private static Cloud.GetVersionResponse getVersionFromMeta(Cloud.GetVersionRequest req) throws RpcException { long startAt = System.nanoTime(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index e398dee9a20254..6b153f25ba0c3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -48,6 +48,7 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import com.google.common.collect.BiMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -746,27 +747,13 @@ public Map getAndCopyPartitionItems() { getDbName(), getName(), getPartitionColumnTypes()); Map res = Maps.newHashMap(); Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); + BiMap idToName = hivePartitionValues.getPartitionNameToIdMap().inverse(); for (Entry entry : idToPartitionItem.entrySet()) { - try { - res.put(getPartitionName(entry.getKey()), entry.getValue()); - } catch (AnalysisException e) { - LOG.info("can not get partitionName by: " + entry.getKey()); - } - + res.put(idToName.get(entry.getKey()), entry.getValue()); } return res; } - @Override - public String getPartitionName(long partitionId) throws AnalysisException { - Map partitionIdToNameMap = getHivePartitionValues().getPartitionIdToNameMap(); - String name = partitionIdToNameMap.get(partitionId); - if (name != null) { - return name; - } - throw new AnalysisException("can not find partition, partitionId: " + partitionId); - } - private HiveMetaStoreCache.HivePartitionValues getHivePartitionValues() { HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .getMetaStoreCache((HMSExternalCatalog) getCatalog()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 379b7fa8ddde2b..b27cc7e93cc521 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -55,6 +55,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -247,7 +249,7 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { LOG.debug("load #{} partitions for {} in catalog {}", partitionNames.size(), key, catalog.getName()); } Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); - Map partitionNameToIdMap = Maps.newHashMapWithExpectedSize(partitionNames.size()); + BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); Map> idToUniqueIdsMap = Maps.newHashMapWithExpectedSize(partitionNames.size()); long idx = 0; for (String partitionName : partitionNames) { @@ -636,7 +638,6 @@ public void addPartitionsCache(String dbName, String tblName, List parti HivePartitionValues copy = partitionValues.copy(); Map idToPartitionItemBefore = copy.getIdToPartitionItem(); Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map partitionIdToNameMapBefore = copy.getPartitionIdToNameMap(); Map> idToUniqueIdsMap = copy.getIdToUniqueIdsMap(); Map idToPartitionItem = new HashMap<>(); long idx = copy.getNextPartitionId(); @@ -650,7 +651,6 @@ public void addPartitionsCache(String dbName, String tblName, List parti idToPartitionItemBefore.put(partitionId, listPartitionItem); idToPartitionItem.put(partitionId, listPartitionItem); partitionNameToIdMapBefore.put(partitionName, partitionId); - partitionIdToNameMapBefore.put(partitionId, partitionName); } Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); @@ -1088,8 +1088,7 @@ public static class HiveFileStatus { @Data public static class HivePartitionValues { private long nextPartitionId; - private Map partitionNameToIdMap; - private Map partitionIdToNameMap; + private BiMap partitionNameToIdMap; private Map> idToUniqueIdsMap; private Map idToPartitionItem; private Map> partitionValuesMap; @@ -1108,7 +1107,7 @@ public HivePartitionValues(Map idToPartitionItem, Map, UniqueId> rangeToId, RangeMap singleColumnRangeMap, long nextPartitionId, - Map partitionNameToIdMap, + BiMap partitionNameToIdMap, Map> idToUniqueIdsMap, Map> singleUidToColumnRangeMap, Map> partitionValuesMap) { @@ -1118,7 +1117,6 @@ public HivePartitionValues(Map idToPartitionItem, this.singleColumnRangeMap = singleColumnRangeMap; this.nextPartitionId = nextPartitionId; this.partitionNameToIdMap = partitionNameToIdMap; - this.partitionIdToNameMap = transferToIdName(partitionNameToIdMap); this.idToUniqueIdsMap = idToUniqueIdsMap; this.singleUidToColumnRangeMap = singleUidToColumnRangeMap; this.partitionValuesMap = partitionValuesMap; @@ -1127,8 +1125,7 @@ public HivePartitionValues(Map idToPartitionItem, public HivePartitionValues copy() { HivePartitionValues copy = new HivePartitionValues(); copy.setNextPartitionId(nextPartitionId); - copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : Maps.newHashMap(partitionNameToIdMap)); - copy.setPartitionIdToNameMap(partitionIdToNameMap == null ? null : Maps.newHashMap(partitionIdToNameMap)); + copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap)); copy.setIdToUniqueIdsMap(idToUniqueIdsMap == null ? null : Maps.newHashMap(idToUniqueIdsMap)); copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap)); @@ -1143,13 +1140,5 @@ public HivePartitionValues copy() { } return copy; } - - private Map transferToIdName(Map partitionNameToIdMap) { - if (partitionNameToIdMap == null) { - return null; - } - return partitionNameToIdMap.entrySet().stream() - .collect(Collectors.toMap(entry -> entry.getValue(), entry -> entry.getKey())); - } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 3d77f42a1ccb40..2c0f157020c1f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -41,7 +41,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -166,14 +165,6 @@ public static Map> generateRelatedPartitionDescs(M return result.getDescs(); } - public static List getPartitionNamesByIds(MTMV mtmv, Collection ids) throws AnalysisException { - List res = Lists.newArrayList(); - for (Long partitionId : ids) { - res.add(mtmv.getPartitionName(partitionId)); - } - return res; - } - public static List getPartitionsIdsByNames(MTMV mtmv, List partitions) throws AnalysisException { mtmv.readLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java index 8aee7741cee0a0..c34df750de5bdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java @@ -79,15 +79,6 @@ public interface MTMVRelatedTableIf extends TableIf { */ MTMVSnapshotIf getTableSnapshot() throws AnalysisException; - /** - * getPartitionName - * - * @param partitionId - * @return partitionName - * @throws AnalysisException - */ - String getPartitionName(long partitionId) throws AnalysisException; - /** * Does the current type of table allow timed triggering * diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 8a6bb03646cab4..5f1e19f32846f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -63,6 +63,8 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.base.Preconditions; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; @@ -477,7 +479,7 @@ public void testAddSingleColumnPartitionsCache() { partitionValueCacheKey.getTypes()); HivePartitionValues partitionValues = metaStoreCache.getPartitionValues(partitionValueCacheKey); Assert.assertEquals(partitionValues.getPartitionNameToIdMap().size(), 4); - Assert.assertEquals(partitionValues.getPartitionIdToNameMap().size(), 4); + Assert.assertEquals(partitionValues.getPartitionNameToIdMap().inverse().size(), 4); } @Test @@ -521,7 +523,7 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key, List HiveMetaStoreCache metaStoreCache) { // partition name format: nation=cn/city=beijing Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); - Map partitionNameToIdMap = Maps.newHashMapWithExpectedSize(partitionNames.size()); + BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); Map> idToUniqueIdsMap = Maps.newHashMapWithExpectedSize(partitionNames.size()); long idx = 0; for (String partitionName : partitionNames) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java index c125e548d06ac6..5cb03801d09d41 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java @@ -99,10 +99,6 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = baseSnapshotIf; - mtmv.getPartitionName(anyLong); - minTimes = 0; - result = "name1"; - mtmv.getRefreshSnapshot(); minTimes = 0; result = refreshSnapshot; @@ -123,10 +119,6 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = baseSnapshotIf; - baseOlapTable.getPartitionName(anyLong); - minTimes = 0; - result = "name1"; - refreshSnapshot.equalsWithRelatedPartition(anyString, anyString, (MTMVSnapshotIf) any); minTimes = 0; result = true;