From 3da25a3cb803603b3e96759c95278ed728c06a2d Mon Sep 17 00:00:00 2001 From: xy720 <22125576+xy720@users.noreply.github.com> Date: Fri, 9 Jan 2026 03:26:32 +0800 Subject: [PATCH] [enhancement](recycle bin) optimize the recycle bin to reduce the potential of FE hang (#55753) ### What problem does this PR solve? I found when there are large amount of garbage(about 90000 partitions) in recycle bin, the Fe's table lock will be hold for long time by DynamicPartitionScheduler thread, the stack is like: ``` "recycle bin" #28 daemon prio=5 os_prio=0 cpu=73880509.81ms elapsed=96569.50s allocated=9212M defined_classes=9 tid=0x00007f0b545c1800 nid=0x2f4540 runnable [0x00007f0b251fd000] java.lang.Thread.State: RUNNABLE at org.apache.doris.catalog.CatalogRecycleBin.getSameNamePartitionIdListToErase(CatalogRecycleBin.java:539) - locked <0x000000020d6d6130> (a org.apache.doris.catalog.CatalogRecycleBin) at org.apache.doris.catalog.CatalogRecycleBin.erasePartitionWithSameName(CatalogRecycleBin.java:556) - locked <0x000000020d6d6130> (a org.apache.doris.catalog.CatalogRecycleBin) at org.apache.doris.catalog.CatalogRecycleBin.erasePartition(CatalogRecycleBin.java:510) - locked <0x000000020d6d6130> (a org.apache.doris.catalog.CatalogRecycleBin) at org.apache.doris.catalog.CatalogRecycleBin.runAfterCatalogReady(CatalogRecycleBin.java:1012) at org.apache.doris.common.util.MasterDaemon.runOneCycle(MasterDaemon.java:58) at org.apache.doris.common.util.Daemon.run(Daemon.java:119) Locked ownable synchronizers: - None "DynamicPartitionScheduler" #41 daemon prio=5 os_prio=0 cpu=115405.50ms elapsed=87942.53s allocated=16637M defined_classes=96 tid=0x00007f0b545cc800 nid=0x2f4545 waiting for monitor entry [0x00007f0b247fe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.doris.catalog.CatalogRecycleBin.recyclePartition(CatalogRecycleBin.java:187) - waiting to lock <0x000000020d6d6130> (a org.apache.doris.catalog.CatalogRecycleBin) at org.apache.doris.catalog.OlapTable.dropPartition(OlapTable.java:1164) at org.apache.doris.catalog.OlapTable.dropPartition(OlapTable.java:1207) at org.apache.doris.datasource.InternalCatalog.dropPartitionWithoutCheck(InternalCatalog.java:1895) at org.apache.doris.datasource.InternalCatalog.dropPartition(InternalCatalog.java:1884) at org.apache.doris.catalog.Env.dropPartition(Env.java:3212) at org.apache.doris.clone.DynamicPartitionScheduler.executeDynamicPartition(DynamicPartitionScheduler.java:605) at org.apache.doris.clone.DynamicPartitionScheduler.runAfterCatalogReady(DynamicPartitionScheduler.java:729) at org.apache.doris.common.util.MasterDaemon.runOneCycle(MasterDaemon.java:58) at org.apache.doris.clone.DynamicPartitionScheduler.run(DynamicPartitionScheduler.java:688) ``` The DynamicPartitionScheduler thread is waiting the CatalogRecycleBin thread while the table write lock is holding by itself . In Fe log, you can see the CatalogRecycleBin thread is running something big and cost almost 5~10 mins every run: ``` fe.log.20250907-2:2025-09-07 04:15:50,740 INFO (recycle bin|28) [CatalogRecycleBin.erasePartition():516] erasePartition eraseNum: 0 cost: 375503ms fe.log.20250907-2:2025-09-07 04:23:14,109 INFO (recycle bin|28) [CatalogRecycleBin.erasePartition():516] erasePartition eraseNum: 0 cost: 413369ms fe.log.20250907-2:2025-09-07 04:30:01,187 INFO (recycle bin|28) [CatalogRecycleBin.erasePartition():516] erasePartition eraseNum: 0 cost: 377077ms fe.log.20250907-2:2025-09-07 04:38:22,769 INFO (recycle bin|28) [CatalogRecycleBin.erasePartition():516] erasePartition eraseNum: 0 cost: 471581ms fe.log.20250907-2:2025-09-07 04:45:42,552 INFO (recycle bin|28) [CatalogRecycleBin.erasePartition():516] erasePartition eraseNum: 0 cost: 409782ms fe.log.20250907-2:2025-09-07 04:54:30,825 INFO (recycle bin|28) [CatalogRecycleBin.erasePartition():516] erasePartition eraseNum: 0 cost: 498272ms fe.log.20250907-2:2025-09-07 05:01:36,311 INFO (recycle bin|28) [CatalogRecycleBin.erasePartition():516] erasePartition eraseNum: 0 cost: 395485ms ``` The most costly task of the CatalogRecycleBin thread is erasing the partition with same name: ``` 2025-09-07 04:16:20,884 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[62638463] name: p_2019051116000 0_20190511170000 from table[32976073] from db[682022] 2025-09-07 04:16:20,994 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[62640651] name: p_2019043016000 0_20190430170000 from table[32976073] from db[682022] 2025-09-07 04:16:21,438 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[60264769] name: p_2019051721000 0_20190517220000 from table[32976073] from db[682022] 2025-09-07 04:16:21,787 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[62651922] name: p_2019051015000 0_20190510160000 from table[32976073] from db[682022] 2025-09-07 04:16:21,893 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[59222503] name: p_2019052708000 0_20190527090000 from table[32976073] from db[682022] 2025-09-07 04:16:22,204 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[62656398] name: p_2019051109000 0_20190511100000 from table[32976073] from db[682022] 2025-09-07 04:16:22,430 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[59228497] name: p_2019051812000 0_20190518130000 from table[32976073] from db[682022] 2025-09-07 04:16:22,493 INFO (recycle bin|28) [CatalogRecycleBin.erasePartitionWithSameName():569] erase partition[62658335] name: p_2019051217000 0_20190512180000 from table[32976073] from db[682022] ... ``` This may leads to whole Fe hang because the table lock is used for many threads. Clipboard_Screenshot_1757283600 This commit mainly optimize the logic of recycling the same name meta, adding caches to reduce the time complexity. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/catalog/CatalogRecycleBin.java | 341 ++++--- .../doris/catalog/CatalogRecycleBinTest.java | 839 ++++++++++++++++++ 2 files changed, 1041 insertions(+), 139 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 837442329954d7..5567d5752054cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -35,12 +35,10 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.HashBasedTable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; -import com.google.common.collect.Table.Cell; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; @@ -55,6 +53,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -70,6 +69,16 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { private Map idToPartition; private Map idToRecycleTime; + // Caches below to avoid calculate meta with same name every demon run cycle. + // When the meta is updated, these caches should be updated too. No need to + // persist these caches because they can be recalculated when FE restarting. + // String: -> Set: + Map> dbNameToIds = new ConcurrentHashMap<>(); + // (DbId, TableName) -> Set: + Map, Set> dbIdTableNameToIds = new ConcurrentHashMap<>(); + // (DbId, TblId) -> (PartitionName -> Set:) + Map, Map>> dbTblIdPartitionNameToIds = new ConcurrentHashMap<>(); + // for compatible in the future @SerializedName("u") String unused; @@ -158,6 +167,7 @@ public synchronized boolean recycleDatabase(Database db, Set tableNames, recycleTime = replayRecycleTime; } idToRecycleTime.put(db.getId(), recycleTime); + dbNameToIds.computeIfAbsent(db.getFullName(), k -> ConcurrentHashMap.newKeySet()).add(db.getId()); LOG.info("recycle db[{}-{}], is force drop: {}", db.getId(), db.getFullName(), isForceDrop); return true; } @@ -182,6 +192,8 @@ public synchronized boolean recycleTable(long dbId, Table table, boolean isRepla } idToRecycleTime.put(table.getId(), recycleTime); idToTable.put(table.getId(), tableInfo); + dbIdTableNameToIds.computeIfAbsent(Pair.of(dbId, table.getName()), + k -> ConcurrentHashMap.newKeySet()).add(table.getId()); LOG.info("recycle table[{}-{}], is force drop: {}", table.getId(), table.getName(), isForceDrop); return true; } @@ -200,6 +212,8 @@ public synchronized boolean recyclePartition(long dbId, long tableId, String tab range, listPartitionItem, dataProperty, replicaAlloc, isInMemory, isMutable); idToRecycleTime.put(partition.getId(), System.currentTimeMillis()); idToPartition.put(partition.getId(), partitionInfo); + dbTblIdPartitionNameToIds.computeIfAbsent(Pair.of(dbId, tableId), k -> new ConcurrentHashMap<>()) + .computeIfAbsent(partition.getName(), k -> ConcurrentHashMap.newKeySet()).add(partition.getId()); LOG.info("recycle partition[{}-{}] of table [{}-{}]", partition.getId(), partition.getName(), tableId, tableName); return true; @@ -251,6 +265,12 @@ private synchronized void eraseDatabase(long currentTimeMs, int keepNum) { // erase db dbIter.remove(); idToRecycleTime.remove(entry.getKey()); + + dbNameToIds.computeIfPresent(db.getFullName(), (k, v) -> { + v.remove(db.getId()); + return v.isEmpty() ? null : v; + }); + Env.getCurrentEnv().eraseDatabase(db.getId(), true); LOG.info("erase db[{}]", db.getId()); eraseNum++; @@ -260,10 +280,9 @@ private synchronized void eraseDatabase(long currentTimeMs, int keepNum) { if (keepNum < 0) { return; } - Set dbNames = idToDatabase.values().stream().map(d -> d.getDb().getFullName()) - .collect(Collectors.toSet()); - for (String dbName : dbNames) { - eraseDatabaseWithSameName(dbName, currentTimeMs, keepNum); + for (Map.Entry> entry : dbNameToIds.entrySet()) { + String dbName = entry.getKey(); + eraseDatabaseWithSameName(dbName, currentTimeMs, keepNum, Lists.newArrayList(entry.getValue())); } } finally { watch.stop(); @@ -271,37 +290,9 @@ private synchronized void eraseDatabase(long currentTimeMs, int keepNum) { } } - private synchronized List getSameNameDbIdListToErase(String dbName, int maxSameNameTrashNum) { - Iterator> iterator = idToDatabase.entrySet().iterator(); - List> dbRecycleTimeLists = Lists.newArrayList(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - RecycleDatabaseInfo dbInfo = entry.getValue(); - Database db = dbInfo.getDb(); - if (db.getFullName().equals(dbName)) { - List dbRecycleTimeInfo = Lists.newArrayList(); - dbRecycleTimeInfo.add(entry.getKey()); - dbRecycleTimeInfo.add(idToRecycleTime.get(entry.getKey())); - - dbRecycleTimeLists.add(dbRecycleTimeInfo); - } - } - List dbIdToErase = Lists.newArrayList(); - if (dbRecycleTimeLists.size() <= maxSameNameTrashNum) { - return dbIdToErase; - } - // order by recycle time desc - dbRecycleTimeLists.sort((x, y) -> - (x.get(1).longValue() < y.get(1).longValue()) ? 1 : ((x.get(1).equals(y.get(1))) ? 0 : -1)); - - for (int i = maxSameNameTrashNum; i < dbRecycleTimeLists.size(); i++) { - dbIdToErase.add(dbRecycleTimeLists.get(i).get(0)); - } - return dbIdToErase; - } - - private synchronized void eraseDatabaseWithSameName(String dbName, long currentTimeMs, int maxSameNameTrashNum) { - List dbIdToErase = getSameNameDbIdListToErase(dbName, maxSameNameTrashNum); + private synchronized void eraseDatabaseWithSameName(String dbName, long currentTimeMs, + int maxSameNameTrashNum, List sameNameDbIdList) { + List dbIdToErase = getIdListToEraseByRecycleTime(sameNameDbIdList, maxSameNameTrashNum); for (Long dbId : dbIdToErase) { RecycleDatabaseInfo dbInfo = idToDatabase.get(dbId); if (!isExpireMinLatency(dbId, currentTimeMs)) { @@ -310,13 +301,19 @@ private synchronized void eraseDatabaseWithSameName(String dbName, long currentT eraseAllTables(dbInfo); idToDatabase.remove(dbId); idToRecycleTime.remove(dbId); + + dbNameToIds.computeIfPresent(dbName, (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); + Env.getCurrentEnv().eraseDatabase(dbId, true); LOG.info("erase database[{}] name: {}", dbId, dbName); } } private synchronized boolean isExpireMinLatency(long id, long currentTimeMs) { - return (currentTimeMs - idToRecycleTime.get(id)) > minEraseLatency; + return (currentTimeMs - idToRecycleTime.get(id)) > minEraseLatency || FeConstants.runningUnitTest; } private void eraseAllTables(RecycleDatabaseInfo dbInfo) { @@ -340,14 +337,28 @@ private void eraseAllTables(RecycleDatabaseInfo dbInfo) { iterator.remove(); idToRecycleTime.remove(table.getId()); tableNames.remove(table.getName()); + + dbIdTableNameToIds.computeIfPresent(Pair.of(tableInfo.getDbId(), table.getName()), (k, v) -> { + v.remove(table.getId()); + return v.isEmpty() ? null : v; + }); + Env.getCurrentEnv().getEditLog().logEraseTable(table.getId()); LOG.info("erase db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); } } public synchronized void replayEraseDatabase(long dbId) { - idToDatabase.remove(dbId); + RecycleDatabaseInfo dbInfo = idToDatabase.remove(dbId); idToRecycleTime.remove(dbId); + + if (dbInfo != null) { + dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); + } + Env.getCurrentEnv().eraseDatabase(dbId, false); LOG.info("replay erase db[{}]", dbId); } @@ -373,6 +384,12 @@ private synchronized void eraseTable(long currentTimeMs, int keepNum) { tableIter.remove(); idToRecycleTime.remove(tableId); + dbIdTableNameToIds.computeIfPresent(Pair.of(tableInfo.getDbId(), table.getName()), + (k, v) -> { + v.remove(tableId); + return v.isEmpty() ? null : v; + }); + // log Env.getCurrentEnv().getEditLog().logEraseTable(tableId); LOG.info("erase table[{}]", tableId); @@ -384,19 +401,9 @@ private synchronized void eraseTable(long currentTimeMs, int keepNum) { if (keepNum < 0) { return; } - Map> dbId2TableNames = Maps.newHashMap(); - for (RecycleTableInfo tableInfo : idToTable.values()) { - Set tblNames = dbId2TableNames.get(tableInfo.dbId); - if (tblNames == null) { - tblNames = Sets.newHashSet(); - dbId2TableNames.put(tableInfo.dbId, tblNames); - } - tblNames.add(tableInfo.getTable().getName()); - } - for (Map.Entry> entry : dbId2TableNames.entrySet()) { - for (String tblName : entry.getValue()) { - eraseTableWithSameName(entry.getKey(), tblName, currentTimeMs, keepNum); - } + for (Map.Entry, Set> entry : dbIdTableNameToIds.entrySet()) { + eraseTableWithSameName(entry.getKey().first, entry.getKey().second, currentTimeMs, keepNum, + Lists.newArrayList(entry.getValue())); } } finally { watch.stop(); @@ -404,43 +411,9 @@ private synchronized void eraseTable(long currentTimeMs, int keepNum) { } } - private synchronized List getSameNameTableIdListToErase(long dbId, String tableName, - int maxSameNameTrashNum) { - Iterator> iterator = idToTable.entrySet().iterator(); - List> tableRecycleTimeLists = Lists.newArrayList(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - RecycleTableInfo tableInfo = entry.getValue(); - if (tableInfo.getDbId() != dbId) { - continue; - } - - Table table = tableInfo.getTable(); - if (table.getName().equals(tableName)) { - List tableRecycleTimeInfo = Lists.newArrayList(); - tableRecycleTimeInfo.add(entry.getKey()); - tableRecycleTimeInfo.add(idToRecycleTime.get(entry.getKey())); - - tableRecycleTimeLists.add(tableRecycleTimeInfo); - } - } - List tableIdToErase = Lists.newArrayList(); - if (tableRecycleTimeLists.size() <= maxSameNameTrashNum) { - return tableIdToErase; - } - // order by recycle time desc - tableRecycleTimeLists.sort((x, y) -> - (x.get(1).longValue() < y.get(1).longValue()) ? 1 : ((x.get(1).equals(y.get(1))) ? 0 : -1)); - - for (int i = maxSameNameTrashNum; i < tableRecycleTimeLists.size(); i++) { - tableIdToErase.add(tableRecycleTimeLists.get(i).get(0)); - } - return tableIdToErase; - } - private synchronized void eraseTableWithSameName(long dbId, String tableName, long currentTimeMs, - int maxSameNameTrashNum) { - List tableIdToErase = getSameNameTableIdListToErase(dbId, tableName, maxSameNameTrashNum); + int maxSameNameTrashNum, List sameNameTableIdList) { + List tableIdToErase = getIdListToEraseByRecycleTime(sameNameTableIdList, maxSameNameTrashNum); for (Long tableId : tableIdToErase) { RecycleTableInfo tableInfo = idToTable.get(tableId); if (!isExpireMinLatency(tableId, currentTimeMs)) { @@ -453,6 +426,12 @@ private synchronized void eraseTableWithSameName(long dbId, String tableName, lo idToTable.remove(tableId); idToRecycleTime.remove(tableId); + + dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, tableName), (k, v) -> { + v.remove(tableId); + return v.isEmpty() ? null : v; + }); + Env.getCurrentEnv().getEditLog().logEraseTable(tableId); LOG.info("erase table[{}] name: {} from db[{}]", tableId, tableName, dbId); } @@ -467,6 +446,13 @@ public synchronized void replayEraseTable(long tableId) { // finish drop db, especially in the case of drop db with many tables. return; } + + dbIdTableNameToIds.computeIfPresent(Pair.of(tableInfo.getDbId(), tableInfo.getTable().getName()), + (k, v) -> { + v.remove(tableId); + return v.isEmpty() ? null : v; + }); + Table table = tableInfo.getTable(); if (table.isManagedTable()) { Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, true); @@ -491,6 +477,15 @@ private synchronized void erasePartition(long currentTimeMs, int keepNum) { // erase partition iterator.remove(); idToRecycleTime.remove(partitionId); + + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partition.getName(), (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; + }); // log Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); LOG.info("erase partition[{}]. reason: expired", partitionId); @@ -502,19 +497,12 @@ private synchronized void erasePartition(long currentTimeMs, int keepNum) { if (keepNum < 0) { return; } - com.google.common.collect.Table> dbTblId2PartitionNames = HashBasedTable.create(); - for (RecyclePartitionInfo partitionInfo : idToPartition.values()) { - Set partitionNames = dbTblId2PartitionNames.get(partitionInfo.dbId, partitionInfo.tableId); - if (partitionNames == null) { - partitionNames = Sets.newHashSet(); - dbTblId2PartitionNames.put(partitionInfo.dbId, partitionInfo.tableId, partitionNames); - } - partitionNames.add(partitionInfo.getPartition().getName()); - } - for (Cell> cell : dbTblId2PartitionNames.cellSet()) { - for (String partitionName : cell.getValue()) { - erasePartitionWithSameName(cell.getRowKey(), cell.getColumnKey(), partitionName, currentTimeMs, - keepNum); + for (Map.Entry, Map>> entry : dbTblIdPartitionNameToIds.entrySet()) { + long dbId = entry.getKey().first; + long tableId = entry.getKey().second; + for (Map.Entry> partitionEntry : entry.getValue().entrySet()) { + erasePartitionWithSameName(dbId, tableId, partitionEntry.getKey(), currentTimeMs, keepNum, + Lists.newArrayList(partitionEntry.getValue())); } } } finally { @@ -523,43 +511,9 @@ private synchronized void erasePartition(long currentTimeMs, int keepNum) { } } - private synchronized List getSameNamePartitionIdListToErase(long dbId, long tableId, String partitionName, - int maxSameNameTrashNum) { - Iterator> iterator = idToPartition.entrySet().iterator(); - List> partitionRecycleTimeLists = Lists.newArrayList(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - RecyclePartitionInfo partitionInfo = entry.getValue(); - if (partitionInfo.getDbId() != dbId || partitionInfo.getTableId() != tableId) { - continue; - } - - Partition partition = partitionInfo.getPartition(); - if (partition.getName().equals(partitionName)) { - List partitionRecycleTimeInfo = Lists.newArrayList(); - partitionRecycleTimeInfo.add(entry.getKey()); - partitionRecycleTimeInfo.add(idToRecycleTime.get(entry.getKey())); - - partitionRecycleTimeLists.add(partitionRecycleTimeInfo); - } - } - List partitionIdToErase = Lists.newArrayList(); - if (partitionRecycleTimeLists.size() <= maxSameNameTrashNum) { - return partitionIdToErase; - } - // order by recycle time desc - partitionRecycleTimeLists.sort((x, y) -> - (x.get(1).longValue() < y.get(1).longValue()) ? 1 : ((x.get(1).equals(y.get(1))) ? 0 : -1)); - - for (int i = maxSameNameTrashNum; i < partitionRecycleTimeLists.size(); i++) { - partitionIdToErase.add(partitionRecycleTimeLists.get(i).get(0)); - } - return partitionIdToErase; - } - private synchronized void erasePartitionWithSameName(long dbId, long tableId, String partitionName, - long currentTimeMs, int maxSameNameTrashNum) { - List partitionIdToErase = getSameNamePartitionIdListToErase(dbId, tableId, partitionName, + long currentTimeMs, int maxSameNameTrashNum, List sameNamePartitionIdList) { + List partitionIdToErase = getIdListToEraseByRecycleTime(sameNamePartitionIdList, maxSameNameTrashNum); for (Long partitionId : partitionIdToErase) { RecyclePartitionInfo partitionInfo = idToPartition.get(partitionId); @@ -571,9 +525,18 @@ private synchronized void erasePartitionWithSameName(long dbId, long tableId, St Env.getCurrentEnv().onErasePartition(partition); idToPartition.remove(partitionId); idToRecycleTime.remove(partitionId); + + dbTblIdPartitionNameToIds.computeIfPresent(Pair.of(dbId, tableId), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partitionName, (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; + }); + Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); - LOG.info("erase partition[{}] name: {} from table[{}] from db[{}]", partitionId, partitionName, tableId, - dbId); + LOG.info("erase partition[{}] name: {} from table[{}] from db[{}]", partitionId, + partitionName, tableId, dbId); } } @@ -586,12 +549,35 @@ public synchronized void replayErasePartition(long partitionId) { return; } + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partitionInfo.getPartition().getName(), (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; + }); + Partition partition = partitionInfo.getPartition(); Env.getCurrentEnv().onErasePartition(partition); LOG.info("replay erase partition[{}]", partitionId); } + private synchronized List getIdListToEraseByRecycleTime(List ids, int maxTrashNum) { + List idToErase = Lists.newArrayList(); + if (ids.size() <= maxTrashNum) { + return idToErase; + } + // order by recycle time desc + ids.sort((x, y) -> Long.compare(idToRecycleTime.get(y), idToRecycleTime.get(x))); + + for (int i = maxTrashNum; i < ids.size(); i++) { + idToErase.add(ids.get(i)); + } + return idToErase; + } + public synchronized Database recoverDatabase(String dbName, long dbId) throws DdlException { RecycleDatabaseInfo dbInfo = null; // The recycle time of the force dropped tables and databases will be set to zero, use 1 here to @@ -625,6 +611,11 @@ public synchronized Database recoverDatabase(String dbName, long dbId) throws Dd idToDatabase.remove(db.getId()); idToRecycleTime.remove(db.getId()); + dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); + return db; } @@ -641,6 +632,11 @@ public synchronized Database replayRecoverDatabase(long dbId) { idToDatabase.remove(dbId); idToRecycleTime.remove(dbId); + dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); + return dbInfo.getDb(); } @@ -668,6 +664,11 @@ private void recoverAllTables(RecycleDatabaseInfo dbInfo) throws DdlException { iterator.remove(); idToRecycleTime.remove(table.getId()); tableNames.remove(table.getName()); + + dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, table.getName()), (k, v) -> { + v.remove(table.getId()); + return v.isEmpty() ? null : v; + }); } if (!tableNames.isEmpty()) { @@ -770,6 +771,12 @@ private synchronized boolean innerRecoverTable(Database db, Table table, String idToTable.remove(table.getId()); } idToRecycleTime.remove(table.getId()); + + dbIdTableNameToIds.computeIfPresent(Pair.of(db.getId(), tableName), (k, v) -> { + v.remove(table.getId()); + return v.isEmpty() ? null : v; + }); + if (isReplay) { LOG.info("replay recover table[{}]", table.getId()); } else { @@ -878,6 +885,15 @@ public synchronized void recoverPartition(long dbId, OlapTable table, String par table.updateVisibleVersionAndTime(version, System.currentTimeMillis()); } + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(recoverPartitionInfo.getDbId(), recoverPartitionInfo.getTableId()), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partitionName, (name, idSet) -> { + idSet.remove(recoverPartition.getId()); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; + }); + // log RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", table.getName(), "", partitionName, newPartitionName); @@ -928,6 +944,15 @@ public synchronized void replayRecoverPartition(OlapTable table, long partitionI table.updateVisibleVersionAndTime(version, System.currentTimeMillis()); } + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(recyclePartitionInfo.getDbId(), table.getId()), (pair, partitionMap) -> { + partitionMap.computeIfPresent(recyclePartitionInfo.getPartition().getName(), (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; + }); + LOG.info("replay recover partition[{}]", partitionId); break; } @@ -945,6 +970,11 @@ public synchronized void eraseDatabaseInstantly(long dbId) throws DdlException { idToDatabase.remove(dbId); idToRecycleTime.remove(dbId); + dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); + // log for erase db String dbName = dbInfo.getDb().getName(); LOG.info("erase db[{}]: {}", dbId, dbName); @@ -1000,6 +1030,11 @@ public synchronized void eraseTableInstantly(long tableId) throws DdlException { idToTable.remove(tableId); idToRecycleTime.remove(tableId); + dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, table.getName()), (k, v) -> { + v.remove(tableId); + return v.isEmpty() ? null : v; + }); + // log for erase table String tableName = table.getName(); Env.getCurrentEnv().getEditLog().logEraseTable(tableId); @@ -1042,6 +1077,15 @@ public synchronized void erasePartitionInstantly(long partitionId) throws DdlExc idToPartition.remove(partitionId); idToRecycleTime.remove(partitionId); + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partition.getName(), (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; + }); + // 4. log for erase partition long tableId = partitionInfo.getTableId(); String partitionName = partition.getName(); @@ -1337,6 +1381,7 @@ public void readFieldsWithGson(DataInput in) throws IOException { RecycleDatabaseInfo dbInfo = new RecycleDatabaseInfo(); dbInfo.readFields(in); idToDatabase.put(id, dbInfo); + dbNameToIds.computeIfAbsent(dbInfo.getDb().getFullName(), k -> ConcurrentHashMap.newKeySet()).add(id); } count = in.readInt(); @@ -1345,6 +1390,8 @@ public void readFieldsWithGson(DataInput in) throws IOException { RecycleTableInfo tableInfo = new RecycleTableInfo(); tableInfo = tableInfo.read(in); idToTable.put(id, tableInfo); + dbIdTableNameToIds.computeIfAbsent(Pair.of(tableInfo.getDbId(), tableInfo.getTable().getName()), + k -> ConcurrentHashMap.newKeySet()).add(id); } count = in.readInt(); @@ -1353,6 +1400,10 @@ public void readFieldsWithGson(DataInput in) throws IOException { RecyclePartitionInfo partitionInfo = new RecyclePartitionInfo(); partitionInfo = partitionInfo.read(in); idToPartition.put(id, partitionInfo); + Pair dbTblId = Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()); + dbTblIdPartitionNameToIds.computeIfAbsent(dbTblId, k -> new ConcurrentHashMap<>()) + .computeIfAbsent(partitionInfo.getPartition().getName(), k -> ConcurrentHashMap.newKeySet()) + .add(id); } count = in.readInt(); @@ -1555,4 +1606,16 @@ public RecyclePartitionInfo read(DataInput in) throws IOException { public List getAllDbIds() { return Lists.newArrayList(idToDatabase.keySet()); } + + // only for unit test + public synchronized void clearAll() { + idToDatabase.clear(); + idToTable.clear(); + idToPartition.clear(); + idToRecycleTime.clear(); + dbNameToIds.clear(); + dbIdTableNameToIds.clear(); + dbTblIdPartitionNameToIds.clear(); + LOG.info("Cleared all objects in recycle bin"); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java new file mode 100644 index 00000000000000..20f5c01bea2b8e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java @@ -0,0 +1,839 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; +import org.apache.doris.catalog.MaterializedIndex.IndexState; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.utframe.UtFrameUtils; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class CatalogRecycleBinTest { + + private static String runningDir; + private static Env env; + + @BeforeClass + public static void beforeClass() throws Exception { + FeConstants.runningUnitTest = true; + runningDir = "fe/mocked/CatalogRecycleBinTest/" + UUID.randomUUID() + "/"; + UtFrameUtils.createDorisCluster(runningDir); + } + + @Before + public void setUp() throws Exception { + env = CatalogTestUtil.createTestCatalog(); + Env.getCurrentRecycleBin().clearAll(); + } + + @AfterClass + public static void tearDown() { + File file = new File(runningDir); + file.delete(); + } + + @Test(expected = IllegalStateException.class) + public void testRecycleNotEmptyDatabase() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Set tableNames = Sets.newHashSet(CatalogTestUtil.testTable1); + Set tableIds = Sets.newHashSet(CatalogTestUtil.testTableId1); + + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + Assert.fail("recycle no empty database should fail"); + } + + @Test + public void testRecycleEmptyDatabase() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database emptyDb1 = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + + Set emptyTableNames = Sets.newHashSet(); + Set emptyTableIds = Sets.newHashSet(); + + Assert.assertTrue(recycleBin.recycleDatabase(emptyDb1, emptyTableNames, emptyTableIds, false, false, 0)); + Assert.assertTrue(recycleBin.isRecycleDatabase(CatalogTestUtil.testDbId1)); + } + + @Test + public void testRecycleSameNameDatabase() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + // keep the newest one in recycle bin + Config.max_same_name_catalog_trash_num = 1; + + Database emptyDb1 = new Database(1001, CatalogTestUtil.testDb1); + Database emptyDb2 = new Database(1002, CatalogTestUtil.testDb1); + Database emptyDb3 = new Database(1003, CatalogTestUtil.testDb1); + + Set emptyTableNames = Sets.newHashSet(); + Set emptyTableIds = Sets.newHashSet(); + + Assert.assertTrue(recycleBin.recycleDatabase(emptyDb1, emptyTableNames, emptyTableIds, false, false, 0)); + Assert.assertTrue(recycleBin.recycleDatabase(emptyDb2, emptyTableNames, emptyTableIds, false, false, 0)); + Assert.assertTrue(recycleBin.isRecycleDatabase(1001)); + Assert.assertTrue(recycleBin.isRecycleDatabase(1002)); + + // sleep 0.1 second to make sure the recycle time is different + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Assert.assertTrue(recycleBin.recycleDatabase(emptyDb3, emptyTableNames, emptyTableIds, false, false, 0)); + Assert.assertTrue(recycleBin.isRecycleDatabase(1003)); + + recycleBin.runAfterCatalogReady(); + + // verify that only newest one is left in recycle bin + Assert.assertFalse(recycleBin.isRecycleDatabase(1001)); + Assert.assertFalse(recycleBin.isRecycleDatabase(1002)); + Assert.assertTrue(recycleBin.isRecycleDatabase(1003)); + } + + @Test + public void testForceDropDatabase() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database emptyDb = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + + Set tableNames = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + + Assert.assertTrue(recycleBin.recycleDatabase(emptyDb, tableNames, tableIds, false, true, 0)); + Assert.assertTrue(recycleBin.isRecycleDatabase(CatalogTestUtil.testDbId1)); + + Long recycleTime = recycleBin.getRecycleTimeById(CatalogTestUtil.testDbId1); + Assert.assertNotNull(recycleTime); + Assert.assertEquals(0L, recycleTime.longValue()); + + recycleBin.runAfterCatalogReady(); + // verify that the db has been immediately dropped from recycle bin + Assert.assertFalse(recycleBin.isRecycleDatabase(CatalogTestUtil.testDbId1)); + // verify recycle time is no longer present + Assert.assertNull(recycleBin.getRecycleTimeById(CatalogTestUtil.testDbId1)); + } + + @Test + public void testRecycleTable() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + Assert.assertTrue(recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, false, 0)); + Assert.assertTrue(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1)); + + // test recycling same table again should fail + Assert.assertFalse(recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, false, 0)); + } + + @Test + public void testForceDropTable() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + Assert.assertTrue(recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, true, 0)); + + Long recycleTime = recycleBin.getRecycleTimeById(CatalogTestUtil.testTableId1); + Assert.assertNotNull(recycleTime); + Assert.assertEquals(0L, recycleTime.longValue()); + Assert.assertTrue(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1)); + + recycleBin.runAfterCatalogReady(); + // verify that the table has been immediately dropped from recycle bin + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1)); + // verify recycle time is no longer present + Assert.assertNull(recycleBin.getRecycleTimeById(CatalogTestUtil.testTableId1)); + } + + @Test + public void testRecyclePartition() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + Partition partition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); + + boolean result = recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + Assert.assertTrue(result); + Assert.assertTrue(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, CatalogTestUtil.testPartitionId1)); + + result = recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + // test recycling same partition again should fail + Assert.assertFalse(result); + } + + @Test + public void testRecycleSameNamePartition() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + // keep the newest one in recycle bin + Config.max_same_name_catalog_trash_num = 1; + + int recyclePartitionNum = 1; + do { + MaterializedIndex index = new MaterializedIndex(1005, IndexState.NORMAL); + RandomDistributionInfo distributionInfo = new RandomDistributionInfo(1); + Partition partition = new Partition(recyclePartitionNum, "same name", index, distributionInfo); + boolean result = recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + Assert.assertTrue(result); + Assert.assertTrue(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, recyclePartitionNum)); + } while (recyclePartitionNum++ < 100); + + // sleep 0.1 second to make sure the recycle time is different + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + MaterializedIndex index = new MaterializedIndex(1005, IndexState.NORMAL); + RandomDistributionInfo distributionInfo = new RandomDistributionInfo(1); + Partition partition = new Partition(recyclePartitionNum, "same name", index, distributionInfo); + boolean result = recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + Assert.assertTrue(result); + Assert.assertTrue(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, recyclePartitionNum)); + + recycleBin.runAfterCatalogReady(); + + // verify that only newest one is left in recycle bin + Set dbIds = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + Set partitionIds = Sets.newHashSet(); + recycleBin.getRecycleIds(dbIds, tableIds, partitionIds); + + Assert.assertEquals(0, dbIds.size()); + Assert.assertEquals(0, tableIds.size()); + Assert.assertEquals(1, partitionIds.size()); + Assert.assertTrue(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, recyclePartitionNum)); + } + + @Test + public void testRecoverEmptyDatabase() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database emptyDb = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + + Set tableNames = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + + recycleBin.recycleDatabase(emptyDb, tableNames, tableIds, false, false, 0); + + Database recoveredDb = recycleBin.recoverDatabase(CatalogTestUtil.testDb1, -1); + Assert.assertNotNull(recoveredDb); + Assert.assertEquals(CatalogTestUtil.testDbId1, recoveredDb.getId()); + Assert.assertFalse(recycleBin.isRecycleDatabase(CatalogTestUtil.testDbId1)); + } + + @Test + public void testRecoverDatabaseWithTable() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Set tableNames = db.getTableNames(); + Set tableIds = Sets.newHashSet(db.getTableIds()); + + recycleAllTables(db, recycleBin); + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + + // test recovering database with table + Database recoveredDb = recycleBin.recoverDatabase(CatalogTestUtil.testDb1, -1); + Assert.assertNotNull(recoveredDb); + Assert.assertEquals(CatalogTestUtil.testDbId1, recoveredDb.getId()); + Assert.assertFalse(recycleBin.isRecycleDatabase(CatalogTestUtil.testDbId1)); + Assert.assertTrue(recoveredDb.getTable(CatalogTestUtil.testTableId1).isPresent()); + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1)); + Assert.assertTrue(recoveredDb.getTable(CatalogTestUtil.testTableId2).isPresent()); + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId2)); + // non olap table should not be recovered + Assert.assertFalse(recoveredDb.getTable(CatalogTestUtil.testEsTableId1).isPresent()); + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testEsTableId1)); + } + + @Test + public void testRecoverTable() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, false, 0); + Assert.assertTrue(recycleBin.recoverTable(db, CatalogTestUtil.testTable1, -1, null)); + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1)); + Assert.assertNotNull(db.getTable(CatalogTestUtil.testTableId1)); + } + + @Test + public void testRecoverPartition() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + Partition partition = olapTable.getPartition(CatalogTestUtil.testPartition1); + + recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + + recycleBin.recoverPartition(CatalogTestUtil.testDbId1, olapTable, CatalogTestUtil.testPartition1, -1, null); + Assert.assertFalse(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, CatalogTestUtil.testPartitionId1)); + Assert.assertNotNull(olapTable.getPartition(CatalogTestUtil.testPartition1)); + } + + @Test + public void testGetRecycleIds() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + Partition partition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); + + recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, false, 0); + + recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + + Set dbIds = Sets.newHashSet(); + Set tableIdsResult = Sets.newHashSet(); + Set partitionIds = Sets.newHashSet(); + + recycleBin.getRecycleIds(dbIds, tableIdsResult, partitionIds); + + Assert.assertEquals(0, dbIds.size()); + Assert.assertTrue(tableIdsResult.contains(CatalogTestUtil.testTableId1)); + Assert.assertTrue(partitionIds.contains(CatalogTestUtil.testPartitionId1)); + } + + @Test + public void testAllTabletsInRecycledStatus() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, false, 0); + + // get tablet ids from the table + List recycleTabletIds = Lists.newArrayList(); + for (Partition partition : olapTable.getAllPartitions()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { + for (Tablet tablet : index.getTablets()) { + recycleTabletIds.add(tablet.getId()); + } + } + } + + List nonRecycledTabletIds = Lists.newArrayList(999L, 1000L); + Assert.assertTrue(recycleBin.allTabletsInRecycledStatus(recycleTabletIds)); + Assert.assertFalse(recycleBin.allTabletsInRecycledStatus(nonRecycledTabletIds)); + } + + @Test + public void testEraseDatabaseInstantly() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database emptyDb = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + + Set tableNames = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + + recycleBin.recycleDatabase(emptyDb, tableNames, tableIds, false, false, 0); + recycleBin.eraseDatabaseInstantly(CatalogTestUtil.testDbId1); + Assert.assertFalse(recycleBin.isRecycleDatabase(CatalogTestUtil.testDbId1)); + } + + @Test + public void testEraseTableInstantly() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, false, 0); + recycleBin.eraseTableInstantly(CatalogTestUtil.testTableId1); + + // verify table is no longer in recycle bin + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1)); + } + + @Test + public void testErasePartitionInstantly() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + Partition partition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); + + recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + + recycleBin.erasePartitionInstantly(CatalogTestUtil.testPartitionId1); + + // verify partition is no longer in recycle bin + Assert.assertFalse(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, CatalogTestUtil.testPartitionId1)); + } + + @Test + public void testReplayOperations() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Set tableNames = Sets.newHashSet(db.getTableNames()); + Set tableIds = Sets.newHashSet(db.getTableIds()); + + Optional
table1 = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table1.isPresent()); + Assert.assertTrue(table1.get() instanceof OlapTable); + OlapTable olapTable1 = (OlapTable) table1.get(); + + Partition partition = olapTable1.getPartition(CatalogTestUtil.testPartitionId1); + recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + + recycleAllTables(db, recycleBin); + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + + recycleBin.replayEraseDatabase(CatalogTestUtil.testDbId1); + recycleBin.replayEraseTable(CatalogTestUtil.testTableId1); + recycleBin.replayEraseTable(CatalogTestUtil.testTableId2); + recycleBin.replayEraseTable(CatalogTestUtil.testEsTableId1); + recycleBin.replayErasePartition(CatalogTestUtil.testPartitionId1); + + // verify objects are no longer in recycle bin + Assert.assertFalse(recycleBin.isRecycleDatabase(CatalogTestUtil.testDbId1)); + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1)); + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId2)); + Assert.assertFalse(recycleBin.isRecycleTable(CatalogTestUtil.testDbId1, CatalogTestUtil.testEsTableId1)); + Assert.assertFalse(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, CatalogTestUtil.testPartitionId1)); + } + + @Test + public void testGetInfo() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Set tableNames = Sets.newHashSet(db.getTableNames()); + Set tableIds = Sets.newHashSet(db.getTableIds()); + + Optional
table1 = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table1.isPresent()); + Assert.assertTrue(table1.get() instanceof OlapTable); + OlapTable olapTable1 = (OlapTable) table1.get(); + + Partition partition = olapTable1.getPartition(CatalogTestUtil.testPartitionId1); + recycleBin.recyclePartition( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testTable1, + partition, + null, + null, + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 3), + false, + false + ); + + recycleAllTables(db, recycleBin); + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + + List> info = recycleBin.getInfo(); + Assert.assertNotNull(info); + Assert.assertFalse(info.isEmpty()); + + // verify info contains database information + Set itemTypes = info.stream().map(item -> item.get(0)).collect(Collectors.toSet()); + Assert.assertTrue(itemTypes.contains("Database")); + Assert.assertTrue(itemTypes.contains("Table")); + Assert.assertTrue(itemTypes.contains("Partition")); + } + + @Test + public void testGetDbToRecycleSize() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Set tableNames = Sets.newHashSet(db.getTableNames()); + Set tableIds = Sets.newHashSet(db.getTableIds()); + + recycleAllTables(db, recycleBin); + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + + Map> sizeMap = recycleBin.getDbToRecycleSize(); + Assert.assertNotNull(sizeMap); + Assert.assertTrue(sizeMap.containsKey(CatalogTestUtil.testDbId1)); + + Pair sizes = sizeMap.get(CatalogTestUtil.testDbId1); + Assert.assertNotNull(sizes); + Assert.assertTrue(sizes.first >= 0); + Assert.assertTrue(sizes.second >= 0); + } + + @Test(expected = DdlException.class) + public void testRecoverNonExistentDatabase() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + // try to recover a non-existent database + recycleBin.recoverDatabase("non_existent_db", -1); + } + + @Test(expected = DdlException.class) + public void testRecoverNonExistentTable() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + // try to recover a non-existent table + recycleBin.recoverTable(db, "non_existent_table", -1, null); + } + + @Test(expected = DdlException.class) + public void testRecoverNonExistentPartition() throws Exception { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + // try to recover a non-existent partition + recycleBin.recoverPartition(CatalogTestUtil.testDbId1, olapTable, "non_existent_partition", -1, null); + } + + @Test + public void testAddTabletToInvertedIndex() { + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + + Database db = CatalogTestUtil.createSimpleDb( + CatalogTestUtil.testDbId1, + CatalogTestUtil.testTableId1, + CatalogTestUtil.testPartitionId1, + CatalogTestUtil.testIndexId1, + CatalogTestUtil.testTabletId1, + CatalogTestUtil.testStartVersion + ); + + Optional
table = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table.isPresent()); + Assert.assertTrue(table.get() instanceof OlapTable); + + OlapTable olapTable = (OlapTable) table.get(); + recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable, false, false, 0); + + TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex(); + invertedIndex.clear(); + TabletMeta tabletMeta = invertedIndex.getTabletMeta(CatalogTestUtil.testTabletId1); + Assert.assertNull(tabletMeta); + + recycleBin.addTabletToInvertedIndex(); + + // verify tablets are added to inverted index + tabletMeta = invertedIndex.getTabletMeta(CatalogTestUtil.testTabletId1); + Assert.assertNotNull(tabletMeta); + } + + public void recycleAllTables(Database db, CatalogRecycleBin recycleBin) { + Optional
table1 = db.getTable(CatalogTestUtil.testTableId1); + Assert.assertTrue(table1.isPresent()); + Assert.assertTrue(table1.get() instanceof OlapTable); + OlapTable olapTable1 = (OlapTable) table1.get(); + + Optional
table2 = db.getTable(CatalogTestUtil.testTableId2); + Assert.assertTrue(table2.isPresent()); + Assert.assertTrue(table2.get() instanceof OlapTable); + OlapTable olapTable2 = (OlapTable) table2.get(); + + Optional
table3 = db.getTable(CatalogTestUtil.testEsTableId1); + Assert.assertTrue(table3.isPresent()); + Assert.assertTrue(table3.get() instanceof EsTable); + EsTable esTable = (EsTable) table3.get(); + + db.unregisterTable(CatalogTestUtil.testTableId1); + recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable1, false, false, 0); + + db.unregisterTable(CatalogTestUtil.testTableId2); + recycleBin.recycleTable(CatalogTestUtil.testDbId1, olapTable2, false, false, 0); + + db.unregisterTable(CatalogTestUtil.testEsTableId1); + recycleBin.recycleTable(CatalogTestUtil.testDbId1, esTable, false, false, 0); + } +} +