diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java index 8c39de41d145..35eb77a52637 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -332,7 +331,7 @@ public void close() throws IOException { private void getTransaction(DeletedBlocksTransaction tx, DatanodeDeletedBlockTransactions transactions, Set dnList, Set replicas, - Map> commandStatus) { + Map> commandStatus) { DeletedBlocksTransaction updatedTxn = DeletedBlocksTransaction.newBuilder(tx) .setCount(transactionStatusManager.getOrDefaultRetryCount( @@ -373,7 +372,7 @@ private Boolean checkInadequateReplica(Set replicas, if (!dnList.contains(datanodeDetails)) { DatanodeDetails dnDetail = replica.getDatanodeDetails(); LOG.debug("Skip Container = {}, because DN = {} is not in dnList.", - containerId, dnDetail.getUuid()); + containerId, dnDetail.getID()); return true; } } @@ -426,10 +425,10 @@ public DatanodeDeletedBlockTransactions getTransactions( // Get the CmdStatus status of the aggregation, so that the current // status of the specified transaction can be found faster - Map> commandStatus = + Map> commandStatus = getSCMDeletedBlockTransactionStatusManager() .getCommandStatusByTxId(dnList.stream(). - map(DatanodeDetails::getUuid).collect(Collectors.toSet())); + map(DatanodeDetails::getID).collect(Collectors.toSet())); ArrayList txIDs = new ArrayList<>(); metrics.setNumBlockDeletionTransactionDataNodes(dnList.size()); Table.KeyValue keyValue = null; @@ -510,7 +509,7 @@ public void setScmCommandTimeoutMs(long scmCommandTimeoutMs) { public void recordTransactionCreated(DatanodeID dnId, long scmCmdId, Set dnTxSet) { getSCMDeletedBlockTransactionStatusManager() - .recordTransactionCreated(dnId.getUuid(), scmCmdId, dnTxSet); + .recordTransactionCreated(dnId, scmCmdId, dnTxSet); } @Override @@ -520,7 +519,7 @@ public int getTransactionToDNsCommitMapSize() { @Override public void onDatanodeDead(DatanodeID dnId) { - getSCMDeletedBlockTransactionStatusManager().onDatanodeDead(dnId.getUuid()); + getSCMDeletedBlockTransactionStatusManager().onDatanodeDead(dnId); } @Override @@ -546,7 +545,7 @@ public void onMessage( ContainerBlocksDeletionACKProto ackProto = commandStatus.getBlockDeletionAck(); getSCMDeletedBlockTransactionStatusManager() - .commitTransactions(ackProto.getResultsList(), dnId.getUuid()); + .commitTransactions(ackProto.getResultsList(), dnId); metrics.incrBlockDeletionCommandSuccess(); metrics.incrDNCommandsSuccess(dnId, 1); } else if (status == CommandStatus.Status.FAILED) { @@ -558,7 +557,7 @@ public void onMessage( } getSCMDeletedBlockTransactionStatusManager() - .commitSCMCommandStatus(deleteBlockStatus.getCmdStatus(), dnId.getUuid()); + .commitSCMCommandStatus(deleteBlockStatus.getCmdStatus(), dnId); } finally { lock.unlock(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java index 4be3a1a57461..cec9c43c6a12 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java @@ -35,10 +35,10 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatus; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -58,7 +58,7 @@ public class SCMDeletedBlockTransactionStatusManager { private static final Logger LOG = LoggerFactory.getLogger(SCMDeletedBlockTransactionStatusManager.class); // Maps txId to set of DNs which are successful in committing the transaction - private final Map> transactionToDNsCommitMap; + private final Map> transactionToDNsCommitMap; // Maps txId to its retry counts; private final Map transactionToRetryCountMap; // The access to DeletedBlocksTXTable is protected by @@ -100,7 +100,7 @@ public SCMDeletedBlockTransactionStatusManager( protected static class SCMDeleteBlocksCommandStatusManager { private static final Logger LOG = LoggerFactory.getLogger(SCMDeleteBlocksCommandStatusManager.class); - private final Map> scmCmdStatusRecord; + private final Map> scmCmdStatusRecord; private static final CmdStatus DEFAULT_STATUS = TO_BE_SENT; private static final Set STATUSES_REQUIRING_TIMEOUT = @@ -128,14 +128,14 @@ public enum CmdStatus { } protected static final class CmdStatusData { - private final UUID dnId; + private final DatanodeID dnId; private final long scmCmdId; private final Set deletedBlocksTxIds; private Instant updateTime; private CmdStatus status; private CmdStatusData( - UUID dnId, long scmTxID, Set deletedBlocksTxIds) { + DatanodeID dnId, long scmTxID, Set deletedBlocksTxIds) { this.dnId = dnId; this.scmCmdId = scmTxID; this.deletedBlocksTxIds = deletedBlocksTxIds; @@ -146,7 +146,7 @@ public Set getDeletedBlocksTxIds() { return Collections.unmodifiableSet(deletedBlocksTxIds); } - public UUID getDnId() { + public DatanodeID getDnId() { return dnId; } @@ -180,7 +180,7 @@ public String toString() { } protected static CmdStatusData createScmCmdStatusData( - UUID dnId, long scmCmdId, Set deletedBlocksTxIds) { + DatanodeID dnId, long scmCmdId, Set deletedBlocksTxIds) { return new CmdStatusData(dnId, scmCmdId, deletedBlocksTxIds); } @@ -190,22 +190,22 @@ protected void recordScmCommand(CmdStatusData statusData) { new ConcurrentHashMap<>()).put(statusData.getScmCmdId(), statusData); } - protected void onSent(UUID dnId, long scmCmdId) { + protected void onSent(DatanodeID dnId, long scmCmdId) { updateStatus(dnId, scmCmdId, CommandStatus.Status.PENDING); } - protected void onDatanodeDead(UUID dnId) { + protected void onDatanodeDead(DatanodeID dnId) { LOG.info("Clean SCMCommand record for Datanode: {}", dnId); scmCmdStatusRecord.remove(dnId); } - protected void updateStatusByDNCommandStatus(UUID dnId, long scmCmdId, + protected void updateStatusByDNCommandStatus(DatanodeID dnId, long scmCmdId, CommandStatus.Status newState) { updateStatus(dnId, scmCmdId, newState); } protected void cleanAllTimeoutSCMCommand(long timeoutMs) { - for (UUID dnId : scmCmdStatusRecord.keySet()) { + for (DatanodeID dnId : scmCmdStatusRecord.keySet()) { for (CmdStatus status : STATUSES_REQUIRING_TIMEOUT) { removeTimeoutScmCommand( dnId, getScmCommandIds(dnId, status), timeoutMs); @@ -213,14 +213,14 @@ protected void cleanAllTimeoutSCMCommand(long timeoutMs) { } } - public void cleanTimeoutSCMCommand(UUID dnId, long timeoutMs) { + public void cleanTimeoutSCMCommand(DatanodeID dnId, long timeoutMs) { for (CmdStatus status : STATUSES_REQUIRING_TIMEOUT) { removeTimeoutScmCommand( dnId, getScmCommandIds(dnId, status), timeoutMs); } } - private Set getScmCommandIds(UUID dnId, CmdStatus status) { + private Set getScmCommandIds(DatanodeID dnId, CmdStatus status) { Set scmCmdIds = new HashSet<>(); Map record = scmCmdStatusRecord.get(dnId); if (record == null) { @@ -234,7 +234,7 @@ private Set getScmCommandIds(UUID dnId, CmdStatus status) { return scmCmdIds; } - private Instant getUpdateTime(UUID dnId, long scmCmdId) { + private Instant getUpdateTime(DatanodeID dnId, long scmCmdId) { Map record = scmCmdStatusRecord.get(dnId); if (record == null || record.get(scmCmdId) == null) { return null; @@ -242,7 +242,7 @@ private Instant getUpdateTime(UUID dnId, long scmCmdId) { return record.get(scmCmdId).getUpdateTime(); } - private void updateStatus(UUID dnId, long scmCmdId, + private void updateStatus(DatanodeID dnId, long scmCmdId, CommandStatus.Status newStatus) { Map recordForDn = scmCmdStatusRecord.get(dnId); if (recordForDn == null) { @@ -309,7 +309,7 @@ private void updateStatus(UUID dnId, long scmCmdId, } } - private void removeTimeoutScmCommand(UUID dnId, + private void removeTimeoutScmCommand(DatanodeID dnId, Set scmCmdIds, long timeoutMs) { Instant now = Instant.now(); for (Long scmCmdId : scmCmdIds) { @@ -323,7 +323,7 @@ private void removeTimeoutScmCommand(UUID dnId, } } - private CmdStatusData removeScmCommand(UUID dnId, long scmCmdId) { + private CmdStatusData removeScmCommand(DatanodeID dnId, long scmCmdId) { Map record = scmCmdStatusRecord.get(dnId); if (record == null || record.get(scmCmdId) == null) { return null; @@ -333,12 +333,12 @@ private CmdStatusData removeScmCommand(UUID dnId, long scmCmdId) { return statusData; } - public Map> getCommandStatusByTxId( - Set dnIds) { - Map> result = + public Map> getCommandStatusByTxId( + Set dnIds) { + Map> result = new HashMap<>(scmCmdStatusRecord.size()); - for (UUID dnId : dnIds) { + for (DatanodeID dnId : dnIds) { Map record = scmCmdStatusRecord.get(dnId); if (record == null) { continue; @@ -361,7 +361,7 @@ private void clear() { } @VisibleForTesting - Map> getScmCmdStatusRecord() { + Map> getScmCmdStatusRecord() { return scmCmdStatusRecord; } } @@ -401,16 +401,16 @@ public int getOrDefaultRetryCount(long txID, int defaultValue) { public void onSent(DatanodeDetails dnId, SCMCommand scmCommand) { scmDeleteBlocksCommandStatusManager.onSent( - dnId.getUuid(), scmCommand.getId()); + dnId.getID(), scmCommand.getId()); } - public Map> getCommandStatusByTxId( - Set dnIds) { + public Map> getCommandStatusByTxId( + Set dnIds) { return scmDeleteBlocksCommandStatusManager.getCommandStatusByTxId(dnIds); } public void recordTransactionCreated( - UUID dnId, long scmCmdId, Set dnTxSet) { + DatanodeID dnId, long scmCmdId, Set dnTxSet) { scmDeleteBlocksCommandStatusManager.recordScmCommand( SCMDeleteBlocksCommandStatusManager .createScmCmdStatusData(dnId, scmCmdId, dnTxSet)); @@ -428,20 +428,20 @@ public void cleanAllTimeoutSCMCommand(long timeoutMs) { scmDeleteBlocksCommandStatusManager.cleanAllTimeoutSCMCommand(timeoutMs); } - public void onDatanodeDead(UUID dnId) { + public void onDatanodeDead(DatanodeID dnId) { scmDeleteBlocksCommandStatusManager.onDatanodeDead(dnId); } public boolean isDuplication(DatanodeDetails dnDetail, long tx, - Map> commandStatus) { - if (alreadyExecuted(dnDetail.getUuid(), tx)) { + Map> commandStatus) { + if (alreadyExecuted(dnDetail.getID(), tx)) { return true; } - return inProcessing(dnDetail.getUuid(), tx, commandStatus); + return inProcessing(dnDetail.getID(), tx, commandStatus); } - public boolean alreadyExecuted(UUID dnId, long txId) { - Set dnsWithTransactionCommitted = + public boolean alreadyExecuted(DatanodeID dnId, long txId) { + Set dnsWithTransactionCommitted = transactionToDNsCommitMap.get(txId); return dnsWithTransactionCommitted != null && dnsWithTransactionCommitted .contains(dnId); @@ -456,10 +456,10 @@ public boolean alreadyExecuted(UUID dnId, long txId) { */ @VisibleForTesting public void commitTransactions( - List transactionResults, UUID dnId) { + List transactionResults, DatanodeID dnId) { ArrayList txIDsToBeDeleted = new ArrayList<>(); - Set dnsWithCommittedTxn; + Set dnsWithCommittedTxn; for (DeleteBlockTransactionResult transactionResult : transactionResults) { if (isTransactionFailed(transactionResult)) { @@ -494,9 +494,9 @@ public void commitTransactions( // the nodes returned in the pipeline match the replication factor. if (min(replicas.size(), dnsWithCommittedTxn.size()) >= container.getReplicationConfig().getRequiredNodes()) { - List containerDns = replicas.stream() + List containerDns = replicas.stream() .map(ContainerReplica::getDatanodeDetails) - .map(DatanodeDetails::getUuid) + .map(DatanodeDetails::getID) .collect(Collectors.toList()); if (dnsWithCommittedTxn.containsAll(containerDns)) { transactionToDNsCommitMap.remove(txID); @@ -527,21 +527,21 @@ public void commitTransactions( @VisibleForTesting public void commitSCMCommandStatus(List deleteBlockStatus, - UUID dnId) { + DatanodeID dnId) { processSCMCommandStatus(deleteBlockStatus, dnId); scmDeleteBlocksCommandStatusManager. cleanTimeoutSCMCommand(dnId, scmCommandTimeoutMs); } - private boolean inProcessing(UUID dnId, long deletedBlocksTxId, - Map> commandStatus) { + private boolean inProcessing(DatanodeID dnId, long deletedBlocksTxId, + Map> commandStatus) { Map deletedBlocksTxStatus = commandStatus.get(dnId); return deletedBlocksTxStatus != null && deletedBlocksTxStatus.get(deletedBlocksTxId) != null; } private void processSCMCommandStatus(List deleteBlockStatus, - UUID dnID) { + DatanodeID dnID) { Map lastStatus = new HashMap<>(); Map summary = new HashMap<>(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java index dad5ce8534ca..b8c9c0f4234f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java @@ -22,10 +22,10 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; import org.apache.hadoop.hdds.scm.PlacementPolicyValidateProxy; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -81,9 +81,9 @@ protected int compareByUsage(DatanodeUsageInfo a, DatanodeUsageInfo b) { if (ret != 0) { return ret; } - UUID uuidA = a.getDatanodeDetails().getUuid(); - UUID uuidB = b.getDatanodeDetails().getUuid(); - return uuidA.compareTo(uuidB); + DatanodeID idA = a.getDatanodeDetails().getID(); + DatanodeID idB = b.getDatanodeDetails().getID(); + return idA.compareTo(idB); } private void setConfiguration(ContainerBalancerConfiguration conf) { @@ -228,8 +228,7 @@ public void increaseSizeEntering(DatanodeDetails target, long size) { } return; } - logger.warn("Cannot find {} in the candidates target nodes", - target.getUuid()); + logger.warn("Cannot find {} in the candidates target nodes", target.getID()); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java index 6388684596eb..9d9638afb11e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java @@ -23,9 +23,9 @@ import java.util.List; import java.util.Map; import java.util.PriorityQueue; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.slf4j.Logger; @@ -56,9 +56,9 @@ public class FindSourceGreedy implements FindSourceStrategy { if (ret != 0) { return ret; } - UUID uuidA = a.getDatanodeDetails().getUuid(); - UUID uuidB = b.getDatanodeDetails().getUuid(); - return uuidA.compareTo(uuidB); + DatanodeID idA = a.getDatanodeDetails().getID(); + DatanodeID idB = b.getDatanodeDetails().getID(); + return idA.compareTo(idB); }); this.nodeManager = nodeManager; } @@ -110,8 +110,7 @@ public void increaseSizeLeaving(DatanodeDetails dui, long size) { addBackSourceDataNode(dui); return; } - LOG.warn("Cannot find datanode {} in candidate source datanodes", - dui.getUuid()); + LOG.warn("Cannot find datanode {} in candidate source datanodes", dui.getID()); } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java index 140d45790fcb..791bee306be7 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java @@ -254,7 +254,7 @@ private void commitTransactions( DatanodeDetails... dns) throws IOException { for (DatanodeDetails dnDetails : dns) { deletedBlockLog.getSCMDeletedBlockTransactionStatusManager() - .commitTransactions(transactionResults, dnDetails.getUuid()); + .commitTransactions(transactionResults, dnDetails.getID()); } scmHADBTransactionBuffer.flush(); } @@ -619,7 +619,7 @@ private void commitSCMCommandStatus(Long scmCmdId, DatanodeID dnID, .getProtoBufMessage()); deletedBlockLog.getSCMDeletedBlockTransactionStatusManager() - .commitSCMCommandStatus(deleteBlockStatus, dnID.getUuid()); + .commitSCMCommandStatus(deleteBlockStatus, dnID); } private void createDeleteBlocksCommandAndAction( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestSCMDeleteBlocksCommandStatusManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestSCMDeleteBlocksCommandStatusManager.java index dfb8ad83b53c..0758b85c4216 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestSCMDeleteBlocksCommandStatusManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestSCMDeleteBlocksCommandStatusManager.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -41,8 +41,8 @@ public class TestSCMDeleteBlocksCommandStatusManager { private SCMDeleteBlocksCommandStatusManager manager; - private UUID dnId1; - private UUID dnId2; + private DatanodeID dnId1; + private DatanodeID dnId2; private long scmCmdId1; private long scmCmdId2; private long scmCmdId3; @@ -56,8 +56,8 @@ public class TestSCMDeleteBlocksCommandStatusManager { public void setup() throws Exception { manager = new SCMDeleteBlocksCommandStatusManager(); // Create test data - dnId1 = UUID.randomUUID(); - dnId2 = UUID.randomUUID(); + dnId1 = DatanodeID.randomID(); + dnId2 = DatanodeID.randomID(); scmCmdId1 = 1L; scmCmdId2 = 2L; scmCmdId3 = 3L; @@ -238,7 +238,7 @@ public void testCleanAllTimeoutSCMCommand() { private void recordAndSentCommand( SCMDeleteBlocksCommandStatusManager statusManager, - UUID dnId, List scmCmdIds, List> txIds) { + DatanodeID dnId, List scmCmdIds, List> txIds) { assertEquals(scmCmdIds.size(), txIds.size()); for (int i = 0; i < scmCmdIds.size(); i++) { long scmCmdId = scmCmdIds.get(i);